{
  "schemaVersion": "crawler-atlas-1",
  "atlasVersion": "atlas-2026-07-28-v1",
  "canonical": "/atlas/url-normalization",
  "title": "URL normalization",
  "category": {
    "slug": "discovery",
    "title": "Discovery and crawl frontiers"
  },
  "definition": "URL normalization maps syntactic variants to a consistent identity before deduplication and scheduling.",
  "mechanism": "Common steps include resolving relative links, removing fragments, normalizing host casing and default ports, and applying a deliberate query-parameter policy.",
  "validation": "Store both the discovered string and normalized fetch target, then test fragments, trailing slashes, parameter order, and percent-encoding independently.",
  "commonFailure": "Over-aggressive normalization can merge distinct resources, while no normalization can turn one page into an effectively infinite crawl space.",
  "related": [
    "/atlas/internal-links",
    "/atlas/canonical-links"
  ],
  "modified": "2026-07-28"
}