{
  "schemaVersion": "crawler-atlas-1",
  "atlasVersion": "atlas-2026-07-28-v1",
  "canonical": "/atlas/crawl-traps",
  "title": "Crawl traps",
  "category": {
    "slug": "quality",
    "title": "Quality, provenance, and observability"
  },
  "definition": "A crawl trap generates an unbounded or extremely large set of low-value URLs.",
  "mechanism": "Calendars, faceted filters, session parameters, recursive paths, and endlessly paginated endpoints can keep producing novel strings for the frontier.",
  "validation": "Track URL-pattern growth, unique-content hashes, depth, and marginal new information; stop test crawlers with explicit budgets.",
  "commonFailure": "Blocking a trap only after millions of variants have been published leaves known URLs in external crawl queues for a long time.",
  "related": [
    "/atlas/pagination",
    "/atlas/duplicate-content"
  ],
  "modified": "2026-07-28"
}