{
  "schemaVersion": "crawler-atlas-1",
  "atlasVersion": "atlas-2026-07-28-v1",
  "canonical": "/atlas/crawl-frontier",
  "title": "The crawl frontier",
  "category": {
    "slug": "discovery",
    "title": "Discovery and crawl frontiers"
  },
  "definition": "The crawl frontier is the ordered queue of discovered URLs that have not yet been fetched.",
  "mechanism": "Schedulers assign priority using signals such as host load, estimated value, freshness, link distance, prior failures, and crawl policy. Different ordering rules can produce different coverage under the same budget.",
  "validation": "Persist discovered, queued, fetched, and parsed timestamps so queue delay and terminal skip reasons can be measured rather than inferred.",
  "commonFailure": "A final list of fetched pages cannot explain whether a missing URL was never discovered, deprioritized, blocked, or exhausted by budget.",
  "related": [
    "/atlas/seed-urls",
    "/atlas/crawl-budget"
  ],
  "modified": "2026-07-28"
}