{
  "schemaVersion": "crawler-atlas-1",
  "atlasVersion": "atlas-2026-07-28-v1",
  "canonical": "/atlas/robots-txt",
  "title": "robots.txt",
  "category": {
    "slug": "directives",
    "title": "Crawler and indexing directives"
  },
  "definition": "robots.txt publishes path-based crawl rules for identified user-agent groups at the origin root.",
  "mechanism": "A compliant crawler retrieves the file, chooses the most specific matching group, and evaluates allow and disallow patterns before requesting a URL.",
  "validation": "Observe the robots request, effective user-agent claim, selected rule, and blocked URL as distinct records; use authentication for actual confidentiality.",
  "commonFailure": "robots.txt is not access control, does not guarantee de-indexing, and may reveal the names of paths a site hoped to keep obscure.",
  "related": [
    "/atlas/meta-robots",
    "/atlas/nofollow"
  ],
  "modified": "2026-07-28"
}