{
  "schemaVersion": 1,
  "slug": "dedup-lookup-coverage",
  "title": "Duplicate lookup coverage: four synthetic cases",
  "summary": "A fixed five-chunk query illustrates missed duplicates with no shared hooks or a one-manifest budget, compared with complete membership.",
  "published": "2026-10-19",
  "updated": "2026-10-19",
  "evidenceType": "synthetic-deterministic-example",
  "license": "CC0-1.0 for Hesela's original implementation and synthetic rows. Cited papers retain their own rights.",
  "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/",
  "sourceUrl": "https://hesela.com/analyses/deduplication-restore-fragmentation/#lookup-coverage",
  "humanPage": "https://hesela.com/analyses/deduplication-restore-fragmentation/#lookup-coverage",
  "downloadPath": "/data/datasets/dedup-lookup-coverage.csv",
  "codePath": "/research/dedup-lookup.mjs",
  "method": "Use older manifest [4,5,6], newer manifest [8,9,10] and query [4,5,8,9,11], with 4096 bytes per chunk. Choose hooks using ID modulo sampleModulo = 0. Repeatedly select the manifest covering most uncovered query hooks, breaking ties toward the later manifest, until the budget or positive scores run out. Count exact query matches in selected manifests. Compare with a complete set-membership baseline. All four existing duplicates are the missed-fraction denominator.",
  "limits": [
    "Hand-picked integer identities are synthetic, not real fingerprints or sampled backup data.",
    "This is a reduced candidate-selection illustration, not a replication or implementation of a complete sparse-indexing system.",
    "No hash collisions, chunking, within-query duplicate IDs, cache, index retention, persistence, compression, placement, timing or failure model.",
    "Loaded manifests are logical selections, not device-read counts. The full-membership baseline has no modeled I/O cost.",
    "Missed duplicates consume additional payload storage; the model does not merge different IDs or test corruption.",
    "Deterministic cases have no sampling uncertainty, confidence interval or observation period; no efficiency estimate for a real workload."
  ],
  "columns": [
    {
      "name": "scenario",
      "type": "string",
      "description": "Synthetic case name; all four cases use the same fixed query and stored manifests."
    },
    {
      "name": "sampleModulo",
      "type": "number",
      "description": "Hook predicate: chunk ID modulo this integer equals zero. Blank for the full-membership baseline; not an empirical sampling rate."
    },
    {
      "name": "maxChampions",
      "type": "number",
      "description": "Maximum selected manifests. Blank for the full-membership baseline."
    },
    {
      "name": "queryHooks",
      "type": "number",
      "description": "Number of qualifying query IDs, including any not found in stored manifests. Blank for the baseline."
    },
    {
      "name": "loadedManifests",
      "type": "number",
      "description": "Number of selected manifests examined; not measured I/O. Blank for baseline because its lookup cost is unspecified."
    },
    {
      "name": "knownDuplicateChunks",
      "type": "number",
      "description": "Incoming distinct chunks already present in any stored manifest, the denominator of missedDuplicateFraction."
    },
    {
      "name": "foundDuplicateChunks",
      "type": "number",
      "description": "Existing query chunks found in examined manifests, or in the complete membership set for baseline."
    },
    {
      "name": "missedDuplicateChunks",
      "type": "number",
      "description": "Existing query chunks outside the selected manifests."
    },
    {
      "name": "newlyStoredChunks",
      "type": "number",
      "description": "Query chunks not found by lookup, including genuinely new chunks and missed duplicates."
    },
    {
      "name": "storedBytes",
      "type": "number",
      "description": "Newly stored payload bytes at 4096 bytes per chunk, excluding metadata, compression and replicas."
    },
    {
      "name": "missedDuplicateFraction",
      "type": "number",
      "description": "Missed existing chunks / known existing chunks, dimensionless. Blank when that denominator is zero."
    }
  ],
  "citations": [
    {
      "id": "lillibridge2009",
      "title": "Lillibridge et al. Sparse Indexing: Large Scale, Inline Deduplication Using Sampling and Locality. USENIX FAST 2009, pp. 111-123.",
      "url": "https://www.usenix.org/legacy/event/fast09/tech/full_papers/lillibridge/lillibridge.pdf",
      "accessed": "2026-10-19"
    },
    {
      "id": "zhu2008",
      "title": "Zhu, Li and Patterson. Avoiding the Disk Bottleneck in the Data Domain Deduplication File System. USENIX FAST 2008, pp. 269-282.",
      "url": "https://www.usenix.org/legacy/event/fast08/tech/full_papers/zhu/zhu.pdf",
      "accessed": "2026-10-19"
    }
  ],
  "generatedAt": "2026-10-19T08:15:59.437Z",
  "inputs": {
    "manifests": [
      {
        "id": "older",
        "chunks": [
          4,
          5,
          6
        ]
      },
      {
        "id": "newer",
        "chunks": [
          8,
          9,
          10
        ]
      }
    ],
    "incoming": [
      4,
      5,
      8,
      9,
      11
    ],
    "chunkBytes": 4096
  },
  "rowCount": 4,
  "rows": [
    {
      "scenario": "no-shared-hook",
      "sampleModulo": 16,
      "maxChampions": 2,
      "queryHooks": 0,
      "loadedManifests": 0,
      "knownDuplicateChunks": 4,
      "foundDuplicateChunks": 0,
      "missedDuplicateChunks": 4,
      "newlyStoredChunks": 5,
      "storedBytes": 20480,
      "missedDuplicateFraction": 1
    },
    {
      "scenario": "one-candidate",
      "sampleModulo": 4,
      "maxChampions": 1,
      "queryHooks": 2,
      "loadedManifests": 1,
      "knownDuplicateChunks": 4,
      "foundDuplicateChunks": 2,
      "missedDuplicateChunks": 2,
      "newlyStoredChunks": 3,
      "storedBytes": 12288,
      "missedDuplicateFraction": 0.5
    },
    {
      "scenario": "two-candidates",
      "sampleModulo": 4,
      "maxChampions": 2,
      "queryHooks": 2,
      "loadedManifests": 2,
      "knownDuplicateChunks": 4,
      "foundDuplicateChunks": 4,
      "missedDuplicateChunks": 0,
      "newlyStoredChunks": 1,
      "storedBytes": 4096,
      "missedDuplicateFraction": 0
    },
    {
      "scenario": "full-membership-baseline",
      "sampleModulo": null,
      "maxChampions": null,
      "queryHooks": null,
      "loadedManifests": null,
      "knownDuplicateChunks": 4,
      "foundDuplicateChunks": 4,
      "missedDuplicateChunks": 0,
      "newlyStoredChunks": 1,
      "storedBytes": 4096,
      "missedDuplicateFraction": 0
    }
  ],
  "artifacts": {
    "modelSha256": "e7e55cdd508de6afa45a5443374c8eeed87c872318ae644895a5b006328893b1",
    "csvSha256": "9a7a47a666b587daa86011c35bb72621a8ed8e0b5f92808e0715a645a4e0086a"
  }
}