{
  "schemaVersion": 1,
  "slug": "hdf5-chunk-cache-reads",
  "title": "HDF5 chunk-cache read observations",
  "summary": "Six cache and access-order cases, repeated three times, measure raw chunk reads for 32 scalar selections from a synthetic compressed HDF5 array.",
  "published": "2026-10-17",
  "updated": "2026-10-17",
  "generatedAt": "2026-10-17T21:07:26.357518+00:00",
  "evidenceType": "measured-library-reads",
  "license": "CC0-1.0 for Hesela's original code and observations; upstream libraries and documents retain their licenses.",
  "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/",
  "humanPage": "https://hesela.com/analyses/hdf5-chunk-cache-read-amplification/",
  "sourceUrl": "https://hesela.com/analyses/hdf5-chunk-cache-read-amplification/",
  "downloadPath": "/data/datasets/hdf5-chunk-cache-reads.csv",
  "rawPath": "/research/hdf5-chunk-cache.json",
  "codePath": "/research/hdf5-chunk-cache.py",
  "requirementsPath": "/research/hdf5-chunk-cache-requirements.txt",
  "generatorPath": "/scripts/build-hdf5-chunk-cache.mjs",
  "rowCount": 18,
  "columns": [
    {
      "name": "repetition",
      "type": "number",
      "description": "One-based repeat on the same host, not an independent performance sample."
    },
    {
      "name": "case",
      "type": "string",
      "description": "Stable case identifier."
    },
    {
      "name": "order",
      "type": "string",
      "description": "Scalar coordinate order: round-robin across chunks, or grouped by chunk."
    },
    {
      "name": "rdccNslots",
      "type": "number",
      "description": "Effective raw-data chunk cache hash-table slots."
    },
    {
      "name": "rdccNbytes",
      "type": "number",
      "description": "Effective raw-data chunk cache byte capacity, excluding other memory."
    },
    {
      "name": "rdccW0",
      "type": "number",
      "description": "Effective eviction parameter, fixed at zero."
    },
    {
      "name": "scalarSelections",
      "type": "number",
      "description": "Number of scalar application reads."
    },
    {
      "name": "selectedBytes",
      "type": "number",
      "description": "Returned application bytes: scalar selections times four."
    },
    {
      "name": "rawChunkReads",
      "type": "number",
      "description": "File-driver read callbacks exactly matching a stored chunk extent."
    },
    {
      "name": "rawChunkBytes",
      "type": "number",
      "description": "Sum of stored payload bytes in those callbacks, excluding metadata."
    },
    {
      "name": "chunkDecodeBytesInferred",
      "type": "number",
      "description": "Arithmetic inference: rawChunkReads times 4096, not measured filter callbacks."
    }
  ],
  "rows": [
    {
      "repetition": 1,
      "case": "one-slot",
      "order": "round-robin",
      "rdccNslots": 1,
      "rdccNbytes": 16384,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 1,
      "case": "more-bytes-one-slot",
      "order": "round-robin",
      "rdccNslots": 1,
      "rdccNbytes": 65536,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 1,
      "case": "four-chunks-fit",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 16384,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 4,
      "rawChunkBytes": 5328,
      "chunkDecodeBytesInferred": 16384
    },
    {
      "repetition": 1,
      "case": "compressed-only-fits",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 2048,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 1,
      "case": "one-chunk-fits",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 4096,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 1,
      "case": "grouped-one-slot",
      "order": "grouped",
      "rdccNslots": 1,
      "rdccNbytes": 4096,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 4,
      "rawChunkBytes": 5328,
      "chunkDecodeBytesInferred": 16384
    },
    {
      "repetition": 2,
      "case": "one-slot",
      "order": "round-robin",
      "rdccNslots": 1,
      "rdccNbytes": 16384,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 2,
      "case": "more-bytes-one-slot",
      "order": "round-robin",
      "rdccNslots": 1,
      "rdccNbytes": 65536,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 2,
      "case": "four-chunks-fit",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 16384,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 4,
      "rawChunkBytes": 5328,
      "chunkDecodeBytesInferred": 16384
    },
    {
      "repetition": 2,
      "case": "compressed-only-fits",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 2048,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 2,
      "case": "one-chunk-fits",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 4096,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 2,
      "case": "grouped-one-slot",
      "order": "grouped",
      "rdccNslots": 1,
      "rdccNbytes": 4096,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 4,
      "rawChunkBytes": 5328,
      "chunkDecodeBytesInferred": 16384
    },
    {
      "repetition": 3,
      "case": "one-slot",
      "order": "round-robin",
      "rdccNslots": 1,
      "rdccNbytes": 16384,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 3,
      "case": "more-bytes-one-slot",
      "order": "round-robin",
      "rdccNslots": 1,
      "rdccNbytes": 65536,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 3,
      "case": "four-chunks-fit",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 16384,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 4,
      "rawChunkBytes": 5328,
      "chunkDecodeBytesInferred": 16384
    },
    {
      "repetition": 3,
      "case": "compressed-only-fits",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 2048,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 3,
      "case": "one-chunk-fits",
      "order": "round-robin",
      "rdccNslots": 17,
      "rdccNbytes": 4096,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 32,
      "rawChunkBytes": 42624,
      "chunkDecodeBytesInferred": 131072
    },
    {
      "repetition": 3,
      "case": "grouped-one-slot",
      "order": "grouped",
      "rdccNslots": 1,
      "rdccNbytes": 4096,
      "rdccW0": 0,
      "scalarSelections": 32,
      "selectedBytes": 128,
      "rawChunkReads": 4,
      "rawChunkBytes": 5328,
      "chunkDecodeBytesInferred": 16384
    }
  ],
  "method": "Fresh HDF5 open per case over the same synthetic in-memory file. Trace read/readinto calls during 32 scalar selections; match exact stored chunk extents; retain metadata reads separately in each step.",
  "limits": [
    "One library build and synthetic trace, not a workload sample or paper replication.",
    "No disk, OS page-cache, latency, throughput, power-loss or device-failure measurement.",
    "Decoded bytes are inferred as raw chunk reads times uncompressed chunk size; filter callbacks were not instrumented.",
    "One slot deliberately forces conflicts; 17 slots is a measured control for this layout, not a universal optimum."
  ],
  "environment": {
    "python": "3.12.3",
    "h5py": "3.15.0",
    "hdf5": "1.14.6",
    "numpy": "2.3.3",
    "zlib": "1.3",
    "system": "Linux",
    "machine": "x86_64",
    "driver": "fileobj over BytesIO",
    "formatBounds": [
      "v110",
      "v110"
    ]
  },
  "fixture": {
    "shape": [
      64,
      64
    ],
    "chunkShape": [
      32,
      32
    ],
    "dtype": "<u4",
    "chunkBytes": 4096,
    "compression": "gzip",
    "compressionLevel": 6,
    "chunks": [
      {
        "coordinate": [
          0,
          0
        ],
        "offset": 475,
        "storedBytes": 1335,
        "filterMask": 0
      },
      {
        "coordinate": [
          0,
          32
        ],
        "offset": 1888,
        "storedBytes": 1334,
        "filterMask": 0
      },
      {
        "coordinate": [
          32,
          0
        ],
        "offset": 3222,
        "storedBytes": 1325,
        "filterMask": 0
      },
      {
        "coordinate": [
          32,
          32
        ],
        "offset": 4547,
        "storedBytes": 1334,
        "filterMask": 0
      }
    ],
    "payloadSha256": "6b0751ba5e64fc9c13ddfb44778fa7d6a1f7d7aa9d6a5e38a1f0a1502c3fb9e3"
  },
  "hashes": {
    "sourceSha256": "2e124037d76b31827e10690f7148a4a62e34e7d47e6bba4aa021326bbce60933",
    "rawSha256": "6762f3586aadbb6f01a4d55392856e0413a4270d584bbe02a29f04c969ad95b3",
    "csvSha256": "396634e4c624806b72307aace35f67b7f0e57b004e8144e77eb528fbf60537d8",
    "requirementsSha256": "1562cd45452194a553bb9af837d31e54fb70bb0f83af365693aaa6c672957e1a",
    "generatorSha256": "bd30ffc05f88536ef799d2f4b1db5ea3283049b9054447c79019b3f5d2d12778"
  },
  "citations": [
    {
      "id": "h5-compression",
      "title": "The HDF Group (2015): Improving I/O Performance When Working with HDF5 Compressed Datasets, pp. 7-20",
      "url": "https://support.hdfgroup.org/releases/hdf5/documentation/hdf5_topics/HDF5ImprovingIOPerformanceCompressedDatasets.pdf",
      "accessed": "2026-10-17"
    },
    {
      "id": "h5-chunking",
      "title": "The HDF Group: Chunking in HDF5, chunk caching and filters",
      "url": "https://support.hdfgroup.org/documentation/hdf5-docs/advanced_topics/chunking_in_hdf5.html",
      "accessed": "2026-10-17"
    },
    {
      "id": "h5py-file",
      "title": "h5py 3.15: File objects, Python file-like objects and chunk cache",
      "url": "https://docs.h5py.org/en/3.15.0/high/file.html",
      "accessed": "2026-10-17"
    },
    {
      "id": "h5py-dataset",
      "title": "h5py 3.15: Datasets, chunked storage and filter pipeline",
      "url": "https://docs.h5py.org/en/3.15.0/high/dataset.html",
      "accessed": "2026-10-17"
    },
    {
      "id": "h5py-low",
      "title": "h5py low-level API: DatasetID.get_chunk_info_by_coord",
      "url": "https://api.h5py.org/h5d.html",
      "accessed": "2026-10-17"
    }
  ]
}