{
  "schemaVersion": 1,
  "slug": "parquet-integrity-boundaries",
  "title": "Parquet checksum and read-coverage observations",
  "summary": "9 read cases across 3 repetitions show how checksum presence, verification and selected columns or row groups change detection of one deliberate payload bit flip.",
  "repetitions": 3,
  "casesPerRepetition": 9,
  "published": "2026-10-17",
  "updated": "2026-10-17",
  "generatedAt": "2026-10-17T19:09:34.743874+00:00",
  "evidenceType": "measured-api-behavior",
  "environment": {
    "python": "3.12.3",
    "pyarrow": "21.0.0",
    "arrowCpp": "21.0.0",
    "kernel": "6.8.0-146-generic",
    "system": "Linux",
    "machine": "x86_64"
  },
  "method": "Write two deterministic fixtures with/without page CRC, flip one payload bit in copies, compare nine read scopes/settings against complete expected selected values; repeat on fresh temporary files.",
  "license": "CC0-1.0 for Hesela's original harness and observations; Apache libraries and documentation retain their own licenses.",
  "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/",
  "humanPage": "https://hesela.com/analyses/parquet-checksums-read-coverage/",
  "sourceUrl": "https://hesela.com/analyses/parquet-checksums-read-coverage/",
  "downloadPath": "/data/datasets/parquet-integrity-boundaries.csv",
  "codePath": "/research/parquet-integrity.py",
  "rawPath": "/research/parquet-integrity.json",
  "requirementsPath": "/research/parquet-integrity-requirements.txt",
  "generatorPath": "/scripts/build-parquet-integrity.mjs",
  "supplements": [
    {
      "schemaVersion": 1,
      "experiment": "parquet-statistics-pruning",
      "title": "Parquet footer statistics and query-pruning observations",
      "evidenceType": "measured-api-behavior",
      "synthetic": true,
      "generatedAt": "2026-10-17T21:46:50.969826+00:00",
      "method": "Generate 128 synthetic rows; structurally change first-group measurement minimum in plaintext footer from 100000 to 100032; keep all page bytes unchanged. Compare CRC-enabled scans and filtered reads, missing/wide-statistics controls, independent page CRCs and an altered-page control.",
      "limits": "One implementation and host; deliberately selected, structurally valid but semantically invalid metadata. Not a spontaneous bit-error model, prevalence estimate, security exploit, benchmark or test of encrypted, compressed, nested or dictionary files. Repetitions are not independent reliability trials.",
      "environment": {
        "python": "3.12.3",
        "packages": {
          "pyarrow": "21.0.0",
          "thriftpy2": "0.5.3",
          "ply": "3.11",
          "six": "1.17.0"
        },
        "arrowCpp": "21.0.0",
        "system": "Linux",
        "kernel": "6.8.0-146-generic",
        "machine": "x86_64"
      },
      "rowCount": 21,
      "columns": [
        {
          "name": "repetition",
          "description": "Fresh deterministic fixture repeat, not an independent reliability trial."
        },
        {
          "name": "case",
          "description": "Read or control case; details in the harness and raw evidence."
        },
        {
          "name": "rows",
          "description": "Returned rows; empty if CRC error prevents a result."
        },
        {
          "name": "expectedRows",
          "description": "Expected rows from the generated input and requested operation."
        },
        {
          "name": "matchesOracle",
          "description": "Whether returned values equal the expected input-derived table; false also for the deliberate CRC-error control."
        },
        {
          "name": "checksumVerification",
          "description": "Reader page checksum verification is enabled in every case."
        }
      ],
      "codePath": "/research/parquet-statistics.py",
      "rawPath": "/research/parquet-statistics.json",
      "csvPath": "/research/parquet-statistics.csv",
      "requirementsPath": "/research/parquet-statistics-requirements.txt",
      "metadataPath": "/research/parquet-statistics-metadata.json",
      "generatorPath": "/scripts/build-parquet-statistics.mjs",
      "schemaUrl": "https://raw.githubusercontent.com/apache/parquet-format/apache-parquet-format-2.10.0/src/main/thrift/parquet.thrift",
      "schemaSha256": "f3152ab8a2574c96f4b0ea3df7694f9170e6a6c18eb9888505115f9acd5e54ce",
      "license": "CC0-1.0 for Hesela's original code and observations; dependencies and downloaded Apache schema retain their own licenses.",
      "hashes": {
        "sourceSha256": "35714c7cdab27c23116e5b965a468e5eddf305e1ff407b1801da2862fbb4469e",
        "requirementsSha256": "24e25a8ab3f0e5677014e1e1f56bce17ba692b891626fcef975b776e31a63300",
        "rawSha256": "5034e039f8479c3c31a2c435c4f45f3821548a0581cdf7e0338b57da5138f4bc",
        "csvSha256": "c22e4242932b8a3146feba97d4e094b2c33743fceec441b6d26a5408cc4ea501"
      }
    }
  ],
  "rowCount": 27,
  "rows": [
    {
      "repetition": 1,
      "case": "clean-checked",
      "verification": true,
      "scope": "all-data",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 1,
      "case": "corrupt-default",
      "verification": "default",
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 1,
      "case": "corrupt-check-off",
      "verification": false,
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 1,
      "case": "corrupt-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "checksum-error",
      "rows": null,
      "mismatchCount": null
    },
    {
      "repetition": 1,
      "case": "projection-check-on",
      "verification": true,
      "scope": "id-only",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 1,
      "case": "other-group-check-on",
      "verification": true,
      "scope": "row-group-1",
      "outcome": "matches-selected-data",
      "rows": 64,
      "mismatchCount": 0
    },
    {
      "repetition": 1,
      "case": "metadata-only",
      "verification": "not-applicable",
      "scope": "metadata",
      "outcome": "metadata-readable",
      "rows": 128,
      "mismatchCount": null
    },
    {
      "repetition": 1,
      "case": "absent-crc-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 1,
      "case": "clean-no-crc-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 2,
      "case": "clean-checked",
      "verification": true,
      "scope": "all-data",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 2,
      "case": "corrupt-default",
      "verification": "default",
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 2,
      "case": "corrupt-check-off",
      "verification": false,
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 2,
      "case": "corrupt-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "checksum-error",
      "rows": null,
      "mismatchCount": null
    },
    {
      "repetition": 2,
      "case": "projection-check-on",
      "verification": true,
      "scope": "id-only",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 2,
      "case": "other-group-check-on",
      "verification": true,
      "scope": "row-group-1",
      "outcome": "matches-selected-data",
      "rows": 64,
      "mismatchCount": 0
    },
    {
      "repetition": 2,
      "case": "metadata-only",
      "verification": "not-applicable",
      "scope": "metadata",
      "outcome": "metadata-readable",
      "rows": 128,
      "mismatchCount": null
    },
    {
      "repetition": 2,
      "case": "absent-crc-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 2,
      "case": "clean-no-crc-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 3,
      "case": "clean-checked",
      "verification": true,
      "scope": "all-data",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 3,
      "case": "corrupt-default",
      "verification": "default",
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 3,
      "case": "corrupt-check-off",
      "verification": false,
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 3,
      "case": "corrupt-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "checksum-error",
      "rows": null,
      "mismatchCount": null
    },
    {
      "repetition": 3,
      "case": "projection-check-on",
      "verification": true,
      "scope": "id-only",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    },
    {
      "repetition": 3,
      "case": "other-group-check-on",
      "verification": true,
      "scope": "row-group-1",
      "outcome": "matches-selected-data",
      "rows": 64,
      "mismatchCount": 0
    },
    {
      "repetition": 3,
      "case": "metadata-only",
      "verification": "not-applicable",
      "scope": "metadata",
      "outcome": "metadata-readable",
      "rows": 128,
      "mismatchCount": null
    },
    {
      "repetition": 3,
      "case": "absent-crc-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "wrong-values",
      "rows": 128,
      "mismatchCount": 1
    },
    {
      "repetition": 3,
      "case": "clean-no-crc-check-on",
      "verification": true,
      "scope": "all-data",
      "outcome": "matches-selected-data",
      "rows": 128,
      "mismatchCount": 0
    }
  ],
  "columns": [
    {
      "name": "repetition",
      "type": "integer",
      "description": "One-based repeat of the same deterministic experiment; not an independent reliability trial."
    },
    {
      "name": "case",
      "type": "string",
      "description": "Controlled read case identifier."
    },
    {
      "name": "verification",
      "type": "boolean|string",
      "description": "Page checksum verification: true, false, default (false in PyArrow 21.0.0), or not-applicable for metadata-only inspection."
    },
    {
      "name": "scope",
      "type": "string",
      "description": "Selected data or metadata; success describes only this scope."
    },
    {
      "name": "outcome",
      "type": "string",
      "description": "Observed return or checksum exception, compared with expected selected payload."
    },
    {
      "name": "rows",
      "type": "integer|null",
      "description": "Returned rows, or declared rows for metadata; empty when checksum error prevents a result."
    },
    {
      "name": "mismatchCount",
      "type": "integer|null",
      "description": "Number of differing selected cells; empty for metadata or an error, not a claim of zero errors."
    }
  ],
  "limits": [
    "Controlled synthetic one-bit fault on one host and pinned library; not field corruption frequency, power-loss testing, a storage benchmark, hostile-tampering assessment or exhaustive file validation. No compressed, encrypted, nested or dictionary pages tested. Projection and row-group cases compare only selected data.",
    "The same chosen bit and input are repeated; no population rate, detection probability or confidence interval is estimated.",
    "A successful checked read does not certify that CRCs existed on every page or that unselected data was validated."
  ],
  "hashes": {
    "sourceSha256": "2c442b6681f860c700b0ce44d877190c77a78a87f7568428d2db753c3b8edde8",
    "requirementsSha256": "0bbd7856f6ec73eebf6f4e98df031a4ac802d738d12735cceaa113b00ab48c19",
    "rawSha256": "c5381e3b338924b6d1fc51460dd4e4a4247be6c3a08cec53bf8995c1f8f8c2dc",
    "csvSha256": "a0c324815a49e0fbb8cd7e5536149cdcfde5015c88996e527d5b941168ea2b6c"
  },
  "citations": [
    {
      "id": "parquet-crc",
      "title": "Apache Parquet: page checksumming",
      "url": "https://parquet.apache.org/docs/file-format/data-pages/checksumming/",
      "accessed": "2026-10-17"
    },
    {
      "id": "arrow-read",
      "title": "Apache Arrow 21.0: read_table checksum verification and column selection",
      "url": "https://arrow.apache.org/docs/21.0/python/generated/pyarrow.parquet.read_table.html",
      "accessed": "2026-10-17"
    },
    {
      "id": "arrow-write",
      "title": "Apache Arrow 21.0: write_table page checksum option",
      "url": "https://arrow.apache.org/docs/21.0/python/generated/pyarrow.parquet.write_table.html",
      "accessed": "2026-10-17"
    },
    {
      "id": "arrow-code",
      "title": "Apache Arrow 21.0.0: SerializedPageReader checksum presence guard",
      "url": "https://github.com/apache/arrow/blob/apache-arrow-21.0.0/cpp/src/parquet/column_reader.cc#L433-L443",
      "accessed": "2026-10-17"
    },
    {
      "id": "parquet-encoding",
      "title": "Apache Parquet: PLAIN INT64 encoding",
      "url": "https://parquet.apache.org/docs/file-format/data-pages/encodings/",
      "accessed": "2026-10-17"
    },
    {
      "id": "parquet-metadata",
      "title": "Apache Parquet: file, column and page metadata",
      "url": "https://parquet.apache.org/docs/file-format/metadata/",
      "accessed": "2026-10-17"
    }
  ]
}