{
  "status": "shipping",
  "advantages": [],
  "supersedes": [],
  "supersededBy": [],
  "firstSeen": null,
  "lastSeen": null,
  "updated": "2026-10-17",
  "slug": "probability-calibration",
  "term": "Probability calibration",
  "aliases": [
    "probabilistic calibration",
    "reliability of probability forecasts"
  ],
  "definition": "Probability calibration is agreement between forecast probabilities and conditional event frequencies among observations receiving those probabilities.",
  "note": "A reliability diagram groups forecast scores and compares mean probability with the observed positive fraction; report the group boundaries and counts. This estimates calibration on an evaluation sample, not a guarantee for every individual drive. [sklearn-calibration]\n\nA fitted calibrator must be evaluated without reusing its fitting data as the final test set. Guo et al. discuss held-out calibration and temperature scaling in image and document classification, not storage failure prediction. [guo-icml17]",
  "limitations": [
    {
      "text": "Calibration alone does not establish useful ranking: a constant forecast can match a pooled event rate while distinguishing no devices; the linked example uses invented counts only.",
      "citationIds": [
        "hesela-calibration"
      ]
    }
  ],
  "related": [
    "brier-score",
    "average-precision",
    "matthews-correlation-coefficient",
    "prediction-horizon",
    "temporal-validation",
    "class-prevalence",
    "expected-calibration-error"
  ],
  "citations": [
    {
      "id": "sklearn-calibration",
      "title": "scikit-learn: probability calibration, reliability diagrams and limits of Brier loss.",
      "url": "https://scikit-learn.org/stable/modules/calibration.html",
      "accessed": "2026-10-17"
    },
    {
      "id": "guo-icml17",
      "title": "Guo, Pleiss, Sun and Weinberger. On Calibration of Modern Neural Networks. ICML 2017, sections 4-4.2.",
      "url": "https://proceedings.mlr.press/v70/guo17a/guo17a.pdf",
      "accessed": "2026-10-17"
    },
    {
      "id": "hesela-calibration",
      "title": "Hesela: synthetic ranking, decision and calibration example; not measured drive results.",
      "url": "https://hesela.com/analyses/ml-disk-failure-prediction/#probability-calibration",
      "accessed": "2026-10-17"
    }
  ]
}