{
  "max_flag_fpr": 0.2,
  "tracks": {
    "embedding-detection": {
      "benchmark_version": "1.1",
      "entries": [
        {
          "rank": 1,
          "name": "shortkit-frequency",
          "title": "Embedding frequency shortcut (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2307.09829",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.988889,
            "auroc_xor": 0.507222,
            "auroc_variance": 0.538889,
            "auroc": 0.678333,
            "fpr": 0.0,
            "tpr": 0.111111,
            "tpr_delta_0.3": 0.0,
            "tpr_delta_0.6": 0.0,
            "tpr_delta_1.0": 0.333333
          }
        },
        {
          "rank": 2,
          "name": "shortkit-probe",
          "title": "Attribute probe (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1610.01644",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.922778,
            "auroc_xor": 0.512222,
            "auroc_variance": 0.520556,
            "auroc": 0.651852,
            "fpr": 0.011111,
            "tpr": 0.188889,
            "tpr_delta_0.3": 0.033333,
            "tpr_delta_0.6": 0.2,
            "tpr_delta_1.0": 0.333333
          }
        },
        {
          "rank": 3,
          "name": "shortkit-conditional-probe",
          "title": "Conditional probe (ShortKit-ML audit)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.987778,
            "auroc_xor": 0.413333,
            "auroc_variance": 0.537778,
            "auroc": 0.646296,
            "fpr": 0.066667,
            "tpr": 0.366667,
            "tpr_delta_0.3": 0.3,
            "tpr_delta_0.6": 0.433333,
            "tpr_delta_1.0": 0.366667
          }
        },
        {
          "rank": 4,
          "name": "shortkit-hbac",
          "title": "HBAC clustering (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.1016/j.ipm.2021.102519",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.827778,
            "auroc_xor": 0.515,
            "auroc_variance": 0.521667,
            "auroc": 0.621481,
            "fpr": 0.0,
            "tpr": 0.022222,
            "tpr_delta_0.3": 0.0,
            "tpr_delta_0.6": 0.0,
            "tpr_delta_1.0": 0.066667
          }
        },
        {
          "rank": 5,
          "name": "shortkit-equalized-odds",
          "title": "Equalized odds gap (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1610.02413",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.648889,
            "auroc_xor": 0.477778,
            "auroc_variance": 0.521111,
            "auroc": 0.549259,
            "fpr": 0.044444,
            "tpr": 0.1,
            "tpr_delta_0.3": 0.033333,
            "tpr_delta_0.6": 0.066667,
            "tpr_delta_1.0": 0.2
          }
        },
        {
          "rank": 6,
          "name": "shortkit-groupdro",
          "title": "GroupDRO gap (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1911.08731",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.391111,
            "auroc_xor": 0.515556,
            "auroc_variance": 0.394444,
            "auroc": 0.433704,
            "fpr": 0.0,
            "tpr": 0.0,
            "tpr_delta_0.3": 0.0,
            "tpr_delta_0.6": 0.0,
            "tpr_delta_1.0": 0.0
          }
        },
        {
          "rank": 7,
          "name": "shortkit-gce",
          "title": "GCE high-loss samples (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1805.07836",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.350556,
            "auroc_xor": 0.395556,
            "auroc_variance": 0.507778,
            "auroc": 0.417963,
            "fpr": 0.055556,
            "tpr": 0.077778,
            "tpr_delta_0.3": 0.033333,
            "tpr_delta_0.6": 0.066667,
            "tpr_delta_1.0": 0.133333
          }
        },
        {
          "rank": 8,
          "name": "mean-difference",
          "title": "Within-task mean difference",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.4324/9780203771587"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc_linear": 0.991111,
            "auroc_xor": 0.447778,
            "auroc_variance": 0.493333,
            "auroc": 0.644074
          }
        },
        {
          "rank": 9,
          "name": "shortkit-bias-direction-pca",
          "title": "Bias direction (PCA) (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1607.06520",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": true,
          "metrics": {
            "auroc_linear": 0.99,
            "auroc_xor": 0.538889,
            "auroc_variance": 0.56,
            "auroc": 0.696296,
            "fpr": 1.0,
            "tpr": 1.0,
            "tpr_delta_0.3": 1.0,
            "tpr_delta_0.6": 1.0,
            "tpr_delta_1.0": 1.0
          }
        },
        {
          "rank": 10,
          "name": "marginal-probe",
          "title": "Marginal probe (E -> A)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1610.01644"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": true,
          "metrics": {
            "auroc_linear": 0.997778,
            "auroc_xor": 0.57,
            "auroc_variance": 0.468889,
            "auroc": 0.678889,
            "fpr": 0.488889,
            "tpr": 0.622222,
            "tpr_delta_0.3": 0.5,
            "tpr_delta_0.6": 0.633333,
            "tpr_delta_1.0": 0.733333
          }
        },
        {
          "rank": 11,
          "name": "shortkit-geometric",
          "title": "Geometric bias analysis (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2505.12540",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": true,
          "metrics": {
            "auroc_linear": 0.995556,
            "auroc_xor": 0.504444,
            "auroc_variance": 0.484444,
            "auroc": 0.661481,
            "fpr": 0.711111,
            "tpr": 0.777778,
            "tpr_delta_0.3": 0.666667,
            "tpr_delta_0.6": 0.8,
            "tpr_delta_1.0": 0.866667
          }
        },
        {
          "rank": 12,
          "name": "shortkit-statistical",
          "title": "Per-dimension group tests (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.1214/aoms/1177730491",
            "https://doi.org/10.1111/j.2517-6161.1995.tb02031.x",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": true,
          "metrics": {
            "auroc_linear": 0.876667,
            "auroc_xor": 0.583333,
            "auroc_variance": 0.508889,
            "auroc": 0.656296,
            "fpr": 1.0,
            "tpr": 0.988889,
            "tpr_delta_0.3": 1.0,
            "tpr_delta_0.6": 0.966667,
            "tpr_delta_1.0": 1.0
          }
        },
        {
          "rank": 13,
          "name": "shortkit-sis",
          "title": "Sufficient input subsets (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1810.03805",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": true,
          "metrics": {
            "auroc_linear": 0.823333,
            "auroc_xor": 0.511667,
            "auroc_variance": 0.605556,
            "auroc": 0.646852,
            "fpr": 1.0,
            "tpr": 1.0,
            "tpr_delta_0.3": 1.0,
            "tpr_delta_0.6": 1.0,
            "tpr_delta_1.0": 1.0
          }
        },
        {
          "rank": 14,
          "name": "shortkit-demographic-parity",
          "title": "Demographic parity gap (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1412.3756",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": true,
          "metrics": {
            "auroc_linear": 0.738889,
            "auroc_xor": 0.643333,
            "auroc_variance": 0.418889,
            "auroc": 0.60037,
            "fpr": 1.0,
            "tpr": 1.0,
            "tpr_delta_0.3": 1.0,
            "tpr_delta_0.6": 1.0,
            "tpr_delta_1.0": 1.0
          }
        }
      ]
    },
    "embedding-mitigation": {
      "benchmark_version": "1.0",
      "entries": [
        {
          "rank": 1,
          "name": "group-reweighting",
          "title": "Group reweighting",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2110.14503"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.67347,
            "mean_acc": 0.747495,
            "gain_over_erm": 0.30438
          }
        },
        {
          "rank": 2,
          "name": "group-balanced-subsample",
          "title": "Group-balanced subsample (DFR-style)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2110.14503"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.662375,
            "mean_acc": 0.713237,
            "gain_over_erm": 0.293284
          }
        },
        {
          "rank": 3,
          "name": "shortkit-adversarial-debiasing",
          "title": "Adversarial debiasing (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1801.07593",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.481708,
            "mean_acc": 0.682814,
            "gain_over_erm": 0.112617,
            "leakage_auroc": 0.856092
          }
        },
        {
          "rank": 4,
          "name": "inlp-projection",
          "title": "INLP (iterative nullspace projection)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2004.07667"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.479418,
            "mean_acc": 0.674023,
            "gain_over_erm": 0.110327,
            "leakage_auroc": 0.762555
          }
        },
        {
          "rank": 5,
          "name": "shortkit-contrastive-debiasing",
          "title": "Contrastive debiasing (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2203.01517",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.430661,
            "mean_acc": 0.672094,
            "gain_over_erm": 0.06157,
            "leakage_auroc": 0.876261
          }
        },
        {
          "rank": 6,
          "name": "shortkit-last-layer-retraining",
          "title": "Last-layer retraining (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2204.02937",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.371406,
            "mean_acc": 0.655995,
            "gain_over_erm": 0.002315
          }
        },
        {
          "rank": 7,
          "name": "erm-logistic",
          "title": "ERM (logistic regression)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.1109/72.788640"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.369091,
            "mean_acc": 0.656193,
            "gain_over_erm": 0.0
          }
        }
      ]
    },
    "image-detection": {
      "benchmark_version": "1.0",
      "entries": [
        {
          "rank": 1,
          "name": "occlusion-flip-consistency",
          "title": "Occlusion flip consistency",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1311.2901"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc": 1.0,
            "reliance_spearman": 0.785621,
            "pointing_game": 0.837357,
            "fpr": 0.0,
            "tpr": 1.0
          }
        },
        {
          "rank": 2,
          "name": "occlusion-hotspot",
          "title": "Occlusion hotspot",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1311.2901"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc": 0.902778,
            "reliance_spearman": 0.61002,
            "pointing_game": 0.878018
          }
        },
        {
          "rank": 3,
          "name": "input-gradient-hotspot",
          "title": "Input-gradient hotspot",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1312.6034"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc": 0.847222,
            "reliance_spearman": 0.635699,
            "pointing_game": 0.810673
          }
        },
        {
          "rank": 4,
          "name": "saliency-peak-consensus",
          "title": "Saliency-peak consensus",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.1038/s41467-019-08987-4"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "auroc": 0.819444,
            "reliance_spearman": 0.277569,
            "pointing_game": 0.878018
          }
        }
      ]
    },
    "image-mitigation": {
      "benchmark_version": "1.0",
      "entries": [
        {
          "rank": 1,
          "name": "shortkit-explanation-regularization",
          "title": "Explanation regularization, RRR (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1703.03717",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.645535,
            "mean_acc": 0.858714,
            "gain_over_erm": 0.400043
          }
        },
        {
          "rank": 2,
          "name": "group-dro-cnn",
          "title": "Group DRO (shared CNN)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/1911.08731"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.545191,
            "mean_acc": 0.799752,
            "gain_over_erm": 0.299699
          }
        },
        {
          "rank": 3,
          "name": "shortkit-shortcut-masking",
          "title": "Shortcut masking (ShortKit-ML)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.1145/3306618.3314293",
            "https://doi.org/10.64898/2026.04.29.26352053"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.529481,
            "mean_acc": 0.77223,
            "gain_over_erm": 0.283989
          }
        },
        {
          "rank": 4,
          "name": "group-balanced-sampling-cnn",
          "title": "Group-balanced sampling (shared CNN)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://arxiv.org/abs/2110.14503"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.43733,
            "mean_acc": 0.805097,
            "gain_over_erm": 0.191839
          }
        },
        {
          "rank": 5,
          "name": "erm-cnn",
          "title": "ERM (shared CNN)",
          "authors": [
            "ShortKit-ML maintainers"
          ],
          "papers": [
            "https://doi.org/10.1109/72.788640"
          ],
          "baseline": true,
          "stale": false,
          "flag_fpr_too_high": false,
          "metrics": {
            "worst_group_acc": 0.245491,
            "mean_acc": 0.756541,
            "gain_over_erm": 0.0
          }
        }
      ]
    }
  }
}