{
  "schemaVersion": "crawler-atlas-1",
  "atlasVersion": "atlas-2026-07-28-v1",
  "canonical": "/atlas/x-robots-tag",
  "title": "X-Robots-Tag",
  "category": {
    "slug": "directives",
    "title": "Crawler and indexing directives"
  },
  "definition": "X-Robots-Tag carries indexing directives in an HTTP response header.",
  "mechanism": "It can govern non-HTML resources such as PDFs and can target specific crawler names, while using directive semantics similar to meta robots.",
  "validation": "Capture response headers before redirects and after the final response; test resource types that cannot contain an HTML head.",
  "commonFailure": "A proxy or CDN configuration can accidentally apply noindex to an entire path when the intent covered only one file type.",
  "related": [
    "/atlas/meta-robots",
    "/atlas/http-status"
  ],
  "modified": "2026-07-28"
}