{
  "slug": "sparse-indexing",
  "term": "Sparse indexing (deduplication)",
  "aliases": [
    "sparse chunk index"
  ],
  "definition": "Sparse indexing uses sampled chunk fingerprints to select stored segment manifests against which incoming chunks are deduplicated.",
  "note": "Selected fingerprints, called hooks, identify candidate manifests. Matching uses the selected manifests, including unsampled chunks, so some existing duplicates may be stored again. [lillibridge2009]\n\nThe candidate limit bounds manifests examined, not total repository containers or restore reads. Unlike a complete-index Bloom filter, absence from the candidate set does not establish global uniqueness. [zhu2008] [hesela-dedup-restore]",
  "status": "research",
  "advantages": [],
  "limitations": [],
  "related": [
    "deduplication",
    "content-defined-chunking",
    "bloom-filter",
    "chunk-fragmentation",
    "container-capping"
  ],
  "supersedes": [],
  "supersededBy": [],
  "firstSeen": null,
  "lastSeen": null,
  "updated": "2026-10-19",
  "citations": [
    {
      "id": "lillibridge2009",
      "title": "Lillibridge et al. Sparse Indexing: Large Scale, Inline Deduplication Using Sampling and Locality. USENIX FAST 2009, pp. 111-123.",
      "url": "https://www.usenix.org/legacy/event/fast09/tech/full_papers/lillibridge/lillibridge.pdf",
      "accessed": "2026-10-19"
    },
    {
      "id": "zhu2008",
      "title": "Zhu, Li and Patterson. Avoiding the Disk Bottleneck in the Data Domain Deduplication File System. USENIX FAST 2008, pp. 269-282.",
      "url": "https://www.usenix.org/legacy/event/fast08/tech/full_papers/zhu/zhu.pdf",
      "accessed": "2026-10-19"
    },
    {
      "id": "hesela-dedup-restore",
      "title": "Hesela: Deduplication ratio does not predict restore cost",
      "url": "https://hesela.com/analyses/deduplication-restore-fragmentation/",
      "accessed": "2026-10-19"
    }
  ],
  "provenance": [
    "Historical algorithm described at FAST 2009; status describes this research concept, not current product availability.",
    "The linked four-case demonstration uses synthetic identities, not the papers' backup traces."
  ]
}