{
 "source": "Robot Intelligence Index dataset, results as of 2026-11-09",
 "call_threshold": 0.9,
 "gap": {
  "real-real": {
   "n_matchups": 60,
   "n_model_pairs": 21,
   "n_bench_pairs": 28,
   "agree": 0.6999244142101281,
   "agree_unweighted": 0.7166666666666667,
   "lo": 0.3512649867913026,
   "hi": 1.0
  },
  "real-sim": {
   "n_matchups": 260,
   "n_model_pairs": 95,
   "n_bench_pairs": 76,
   "agree": 0.7316261241338646,
   "agree_unweighted": 0.7538461538461538,
   "lo": 0.5881135545873609,
   "hi": 0.8598532594856122
  },
  "sim-sim": {
   "n_matchups": 381,
   "n_model_pairs": 185,
   "n_bench_pairs": 64,
   "agree": 0.93171171171171,
   "agree_unweighted": 0.89501312335958,
   "lo": 0.8688538334787246,
   "hi": 0.9830259991630047
  }
 },
 "robodojo": {
  "sim_to_arm": {
   "agree": 0.935,
   "lo": 0.795,
   "hi": 1.0,
   "n": 107
  },
  "arm_to_arm": {
   "agree": 0.965,
   "lo": 0.868,
   "hi": 1.0,
   "n": 172
  },
  "same_verdict": {
   "k": 13,
   "n": 13,
   "pairs": 6
  },
  "exchange": {
   "sim": {
    "m": 16.4,
    "lo": 5.5,
    "hi": null
   },
   "arm": {
    "m": 22.5,
    "lo": 7.2,
    "hi": null
   }
  }
 },
 "matched": {
  "sim": 0.717,
  "real": 0.733,
  "diff": -0.016,
  "lo": -0.245,
  "hi": 0.123,
  "cases": 46,
  "pairs": 18
 },
 "sim_sim_minus_sim_real": {
  "diff": 0.2,
  "lo": 0.065,
  "hi": 0.356,
  "p": 0.0067
 },
 "noise_only_disagreement": 0.023,
 "benchmarks": {
  "robotwin": {
   "agreement": {
    "n": 28,
    "pairs": 19,
    "models": 10,
    "agree": 0.9473684210526315,
    "a_lo": 0.75,
    "a_hi": 1.0,
    "trials": 36.308640695776525,
    "m_lo": 5.73031070766817,
    "m_hi": NaN,
    "n_rate": 27
   },
   "rank_correlation": {
    "n": 10,
    "rho": 0.6,
    "lo": -0.012987012987012986,
    "hi": 0.9876543209876544
   },
   "verdict": "Tracks real robots"
  },
  "robocoliseum": {
   "agreement": {
    "n": 39,
    "pairs": 27,
    "models": 13,
    "agree": 0.7530864197530863,
    "a_lo": 0.4615384615384616,
    "a_hi": 0.9881349206349206,
    "trials": 2.274832198814337,
    "m_lo": 0.0,
    "m_hi": 42.06815253965369,
    "n_rate": 30
   },
   "rank_correlation": {
    "n": 13,
    "rho": 0.532967032967033,
    "lo": 0.0909090909090909,
    "hi": 0.8238885848570692
   },
   "verdict": "Tracks real robots"
  },
  "efe": {
   "agreement": {
    "n": 8,
    "pairs": 8,
    "models": 6,
    "agree": 0.75,
    "a_lo": 0.4,
    "a_hi": 1.0,
    "trials": 23.363633256652168,
    "m_lo": 0.9791442279447794,
    "m_hi": NaN,
    "n_rate": 7
   },
   "rank_correlation": {
    "n": 6,
    "rho": 0.8857142857142858,
    "lo": 0.5151515151515151,
    "hi": 1.0
   },
   "verdict": "Tracks real robots"
  },
  "robolab": {
   "agreement": {
    "n": 17,
    "pairs": 13,
    "models": 8,
    "agree": 0.7692307692307693,
    "a_lo": 0.35714285714285715,
    "a_hi": 1.0,
    "trials": Infinity,
    "m_lo": 86.0,
    "m_hi": NaN,
    "n_rate": 6
   },
   "rank_correlation": {
    "n": 9,
    "rho": 0.6,
    "lo": -0.029670619235836606,
    "hi": 0.9829059829059827
   },
   "verdict": "Not yet clear"
  },
  "molmospaces": {
   "agreement": {
    "n": 16,
    "pairs": 7,
    "models": 5,
    "agree": 0.75,
    "a_lo": 0.3333333333333333,
    "a_hi": 1.0,
    "trials": 2.978177099390331,
    "m_lo": 0.0,
    "m_hi": NaN,
    "n_rate": 15
   },
   "rank_correlation": {
    "n": 7,
    "rho": 0.28571428571428575,
    "lo": -0.5925925925925927,
    "hi": 0.9629629629629631
   },
   "verdict": "Not yet clear"
  },
  "calvin": {
   "agreement": {
    "n": 15,
    "pairs": 8,
    "models": 5,
    "agree": 0.7500000000000001,
    "a_lo": 0.25,
    "a_hi": 1.0,
    "trials": 4.916681379905454,
    "m_lo": 0.0,
    "m_hi": NaN,
    "n_rate": 14
   },
   "rank_correlation": null,
   "verdict": "Not yet clear"
  },
  "simpler_wx": {
   "agreement": {
    "n": 15,
    "pairs": 8,
    "models": 5,
    "agree": 0.625,
    "a_lo": 0.25,
    "a_hi": 1.0,
    "trials": 0.968057681896248,
    "m_lo": 0.0,
    "m_hi": NaN,
    "n_rate": 14
   },
   "rank_correlation": {
    "n": 6,
    "rho": 0.3714285714285715,
    "lo": -0.4545454545454545,
    "hi": 0.9393939393939392
   },
   "verdict": "Not yet clear"
  },
  "libero_pro2": {
   "agreement": {
    "n": 30,
    "pairs": 11,
    "models": 7,
    "agree": 0.6257575757575763,
    "a_lo": 0.25,
    "a_hi": 0.9583333333333334,
    "trials": 0.5134581298072093,
    "m_lo": 0.0,
    "m_hi": 25.187820106613405,
    "n_rate": 26
   },
   "rank_correlation": {
    "n": 7,
    "rho": 0.21428571428571433,
    "lo": -0.5925925925925927,
    "hi": 0.7647058823529412
   },
   "verdict": "Not yet clear"
  },
  "robodojo_sim": {
   "agreement": {
    "n": 48,
    "pairs": 29,
    "models": 13,
    "agree": 0.6241379310344828,
    "a_lo": 0.3333333333333333,
    "a_hi": 0.8902592592592591,
    "trials": 0.27614523814260605,
    "m_lo": 0.0,
    "m_hi": 8.492633806099713,
    "n_rate": 39
   },
   "rank_correlation": {
    "n": 13,
    "rho": 0.24725274725274726,
    "lo": -0.3351808291678918,
    "hi": 0.770340203120354
   },
   "verdict": "Not yet clear"
  },
  "libero": {
   "agreement": {
    "n": 10,
    "pairs": 7,
    "models": 7,
    "agree": 0.5714285714285714,
    "a_lo": 0.0,
    "a_hi": 1.0,
    "trials": 2.9690873699093667,
    "m_lo": 0.0,
    "m_hi": NaN,
    "n_rate": 9
   },
   "rank_correlation": {
    "n": 9,
    "rho": -0.35,
    "lo": -0.8162589194699286,
    "hi": 0.3333333333333333
   },
   "verdict": "No link found"
  },
  "simpler_g": {
   "agreement": {
    "n": 14,
    "pairs": 7,
    "models": 5,
    "agree": 0.42857142857142866,
    "a_lo": 0.0,
    "a_hi": 1.0,
    "trials": 0.0,
    "m_lo": 0.0,
    "m_hi": NaN,
    "n_rate": 13
   },
   "rank_correlation": {
    "n": 6,
    "rho": -0.3142857142857143,
    "lo": -0.818181818181818,
    "hi": 0.3548387096774194
   },
   "verdict": "No link found"
  }
 },
 "reversed_calls": {
  "n": 64,
  "share99": 0.65625,
  "median_z": 3.2267590948910474
 },
 "interventions": [
  {
   "name": "RoboTwin 2.0: scene randomization",
   "labA": "clean",
   "labB": "randomized",
   "matchups": 28,
   "pairs": 19,
   "models": 10,
   "accA": 0.9210526315789476,
   "accB": 0.9473684210526315,
   "diff": 0.02631578947368396,
   "lo": -0.1,
   "hi": 0.14285714285714285,
   "p_le0": 0.5288024254674077,
   "sdA": 19.333265165256318,
   "sdB": 27.090971695239915,
   "callA": 0.8928571428571429,
   "callB": 0.8928571428571429
  },
  {
   "name": "LIBERO-Pro2.0: perturbations",
   "labA": "unperturbed",
   "labB": "static pert.",
   "matchups": 32,
   "pairs": 13,
   "models": 7,
   "accA": 0.38846153846153847,
   "accB": 0.6064102564102564,
   "diff": 0.2179487179487179,
   "lo": 0.0,
   "hi": 0.47916666666666674,
   "p_le0": 0.12346298619824342,
   "sdA": 14.99209851059815,
   "sdB": 13.547169795475941,
   "callA": 0.65625,
   "callB": 0.9375
  },
  {
   "name": "LIBERO-Pro2.0: live perturbations",
   "labA": "unperturbed",
   "labB": "live pert.",
   "matchups": 32,
   "pairs": 13,
   "models": 7,
   "accA": 0.38846153846153847,
   "accB": 0.6256410256410256,
   "diff": 0.23717948717948717,
   "lo": 0.0,
   "hi": 0.5,
   "p_le0": 0.10589711417816813,
   "sdA": 14.99209851059815,
   "sdB": 16.245092986925194,
   "callA": 0.65625,
   "callB": 0.8125
  },
  {
   "name": "LIBERO: clean (papers) vs Pro2 pert.",
   "labA": "clean (paper)",
   "labB": "perturbed",
   "matchups": 21,
   "pairs": 9,
   "models": 6,
   "accA": 0.44444444444444464,
   "accB": 0.6481481481481484,
   "diff": 0.20370370370370372,
   "lo": -0.21428571428571427,
   "hi": 0.5833333333333335,
   "p_le0": 0.23884782054550088,
   "sdA": 6.7190901794443985,
   "sdB": 11.199107107265293,
   "callA": 0.047619047619047616,
   "callB": 0.9047619047619048
  },
  {
   "name": "LIBERO: clean (papers) vs Plus",
   "labA": "clean (paper)",
   "labB": "LIBERO-Plus",
   "matchups": 2,
   "pairs": 2,
   "models": 3,
   "accA": 0.25,
   "accB": 0.5,
   "diff": 0.25,
   "lo": -0.5,
   "hi": 1.0,
   "p_le0": 0.31353383458646616,
   "sdA": 7.151553832715113,
   "sdB": 18.54691498755401,
   "callA": 0.5,
   "callB": 1.0
  },
  {
   "name": "RoboCasa365: unseen tasks",
   "labA": "atomic seen",
   "labB": "multi unseen",
   "matchups": 7,
   "pairs": 3,
   "models": 3,
   "accA": 0.8333333333333333,
   "accB": 0.5,
   "diff": -0.33333333333333326,
   "lo": -0.5,
   "hi": 0.0,
   "p_le0": 1.0,
   "sdA": 23.165040410545853,
   "sdB": 16.24583707360819,
   "callA": 0.2857142857142857,
   "callB": 0.0
  },
  {
   "name": "RoboCasa365: unseen vs seen multi",
   "labA": "multi seen",
   "labB": "multi unseen",
   "matchups": 7,
   "pairs": 3,
   "models": 3,
   "accA": 0.5,
   "accB": 0.5,
   "diff": 0.0,
   "lo": 0.0,
   "hi": 0.0,
   "p_le0": 1.0,
   "sdA": 24.16491372952851,
   "sdB": 16.24583707360819,
   "callA": 0.0,
   "callB": 0.0
  },
  {
   "name": "MolmoSpaces: random camera",
   "labA": "pick",
   "labB": "pick rand cam",
   "matchups": 6,
   "pairs": 3,
   "models": 3,
   "accA": 0.75,
   "accB": 0.8333333333333334,
   "diff": 0.08333333333333337,
   "lo": 0.0,
   "hi": 0.25,
   "p_le0": 0.4872226902555462,
   "sdA": 20.90166904220809,
   "sdB": 22.15772269435648,
   "callA": 1.0,
   "callB": 0.3333333333333333
  },
  {
   "name": "RoboColiseum: robustness track",
   "labA": "general manip",
   "labB": "robustness",
   "matchups": 45,
   "pairs": 32,
   "models": 13,
   "accA": 0.5416666666666667,
   "accB": 0.6666666666666667,
   "diff": 0.125,
   "lo": -0.09693548387096769,
   "hi": 0.3,
   "p_le0": 0.20375,
   "sdA": 12.003081575704607,
   "sdB": 14.289921830716146,
   "callA": 0.6666666666666666,
   "callB": 0.7555555555555555
  },
  {
   "name": "RoboColiseum: instr. following",
   "labA": "general manip",
   "labB": "instruction",
   "matchups": 45,
   "pairs": 32,
   "models": 13,
   "accA": 0.5416666666666667,
   "accB": 0.6666666666666667,
   "diff": 0.125,
   "lo": -0.07692307692307693,
   "hi": 0.2916666666666667,
   "p_le0": 0.17975,
   "sdA": 12.003081575704607,
   "sdB": 16.32235166846125,
   "callA": 0.6666666666666666,
   "callB": 0.8
  }
 ],
 "fixed_recipe": {
  "fixed": 0.75,
  "dev": 0.59,
  "diff": 0.16,
  "lo": -0.19,
  "hi": 0.48
 },
 "robustness": [
  [
   "As published",
   0.7316261241338646,
   2.8679604170249022
  ],
  [
   "Every noise level halved",
   0.718,
   3.5
  ],
  [
   "Every noise level doubled",
   0.807,
   2.9
  ],
  [
   "Call threshold 80% instead of 90%",
   0.725,
   3.3
  ],
  [
   "Call threshold 95%",
   0.714,
   2.0
  ],
  [
   "No threshold: every matchup, including close calls",
   0.683,
   null
  ]
 ],
 "leave_one_out": {
  "lo": 0.669,
  "hi": 0.777,
  "no_robochallenge": 0.77
 },
 "direct_test": {
  "Genie Sim 3.0": {
   "units": 4,
   "policies": 4,
   "cells": 16,
   "level": 3.25,
   "spearman": 0.9871708245126285,
   "mmrv": 0.004374999999999995,
   "agree": 1.0,
   "n_calls": 12,
   "ci": {
    "level": [
     1.5624999999999993,
     4.937500000000001
    ],
    "spearman": [
     0.9665063509461097,
     1.0
    ],
    "mmrv": [
     0.0,
     0.005833333333333327
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.8972289509511086
  },
  "MolmoBot": {
   "units": 2,
   "policies": 7,
   "cells": 12,
   "level": 1.9166666666666659,
   "spearman": 0.875,
   "mmrv": 0.07614285714285712,
   "agree": 0.9444444444444444,
   "n_calls": 18,
   "ci": {
    "level": [
     -13.328571428571435,
     12.324175824175821
    ],
    "spearman": [
     0.6,
     1.0
    ],
    "mmrv": [
     0.0,
     0.13349999999999998
    ],
    "agree": [
     0.75,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.9307720755408379
  },
  "PolaRiS": {
   "units": 6,
   "policies": 5,
   "cells": 30,
   "level": 18.463056864048927,
   "spearman": 0.9124465724134826,
   "mmrv": 0.02987179487179488,
   "agree": 1.0,
   "n_calls": 31,
   "ci": {
    "level": [
     15.351923077356412,
     21.98393424048503
    ],
    "spearman": [
     0.8110042339640732,
     0.9333333333333332
    ],
    "mmrv": [
     0.0018518518518518571,
     0.036111111111111115
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9998148148148148,
   "rerun_spearman": 0.8732627949295722
  },
  "PolaRiS control": {
   "units": 6,
   "policies": 5,
   "cells": 30,
   "level": -31.839188034188034,
   "spearman": 0.8457799057468159,
   "mmrv": 0.03917735042735043,
   "agree": 1.0,
   "n_calls": 17,
   "ci": {
    "level": [
     -38.91282051282051,
     -23.408333333333328
    ],
    "spearman": [
     0.5610042339640732,
     0.9166666666666666
    ],
    "mmrv": [
     0.0018518518518518571,
     0.04619391025641025
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9996410256410256,
   "rerun_spearman": 0.8742826227548346
  },
  "RoboDojo": {
   "units": 7,
   "policies": 10,
   "cells": 70,
   "level": -1.2751428571428574,
   "spearman": 0.31033877723343256,
   "mmrv": 0.10582857142857142,
   "agree": 0.8157894736842105,
   "n_calls": 76,
   "ci": {
    "level": [
     -3.629121428571429,
     1.0570857142857135
    ],
    "spearman": [
     0.08463964306633465,
     0.587447202795477
    ],
    "mmrv": [
     0.039247857142857154,
     0.11112244897959182
    ],
    "agree": [
     0.5833333333333334,
     0.95
    ]
   },
   "rerun_agree": 0.9988300620755277,
   "rerun_spearman": 0.7406988300613535
  },
  "RoboLab-120 (v2)": {
   "units": 1,
   "policies": 4,
   "cells": 4,
   "level": 9.700000000000001,
   "spearman": 0.7999999999999999,
   "mmrv": 0.14550000000000002,
   "agree": 1.0,
   "n_calls": 4,
   "ci": {
    "level": [
     -3.5749999999999993,
     22.975
    ],
    "spearman": [
     0.5,
     1.0
    ],
    "mmrv": [
     0.0,
     0.19400000000000003
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.9347037225180279
  },
  "SimplerEnv": {
   "units": 18,
   "policies": 7,
   "cells": 92,
   "level": 13.314130434782612,
   "spearman": 0.7974987230404877,
   "mmrv": 0.08802592592592592,
   "agree": 1.0,
   "n_calls": 114,
   "ci": {
    "level": [
     8.625757575757577,
     18.04466666666666
    ],
    "spearman": [
     0.4594611094309768,
     0.9
    ],
    "mmrv": [
     0.019047619047619046,
     0.10169642857142858
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9998824858431158,
   "rerun_spearman": 0.9495042785860568
  },
  "VLA-Replica": {
   "units": 5,
   "policies": 4,
   "cells": 20,
   "level": -1.0,
   "spearman": 0.6523920842049056,
   "mmrv": 0.04,
   "agree": 1.0,
   "n_calls": 6,
   "ci": {
    "level": [
     -4.0,
     2.0
    ],
    "spearman": [
     0.5267949192431123,
     0.875
    ],
    "mmrv": [
     0.013333333333333332,
     0.05333333333333333
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9938690476190476,
   "rerun_spearman": 0.6427474394750752
  },
  "TIER twin": {
   "units": 21,
   "policies": 16,
   "cells": 98,
   "level": 11.226445978790489,
   "spearman": 0.909930814222628,
   "mmrv": 0.03208241758241759,
   "agree": 1.0,
   "n_calls": 107,
   "ci": {
    "level": [
     9.21377049180328,
     16.089265469777075
    ],
    "spearman": [
     0.8074897381616175,
     0.9381578410617949
    ],
    "mmrv": [
     0.008487179487179487,
     0.03590813609467456
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9996299592657306,
   "rerun_spearman": 0.9196639018796149
  },
  "TIER same-name": {
   "units": 10,
   "policies": 21,
   "cells": 86,
   "level": -0.31930232558139554,
   "spearman": 0.490225851488955,
   "mmrv": 0.10385857142857143,
   "agree": 0.8469387755102041,
   "n_calls": 98,
   "ci": {
    "level": [
     -3.2530404842071508,
     2.0074268243785065
    ],
    "spearman": [
     0.29738185211339035,
     0.664802003221041
    ],
    "mmrv": [
     0.04702722222222223,
     0.10500607473544972
    ],
    "agree": [
     0.6817272727272727,
     0.9519459706959706
    ]
   },
   "rerun_agree": 0.9986864021395612,
   "rerun_spearman": 0.8081549190234638
  },
  "TIER control": {
   "units": 13,
   "policies": 11,
   "cells": 70,
   "level": -3.208223443223442,
   "spearman": 0.7495747756204536,
   "mmrv": 0.1032715976331361,
   "agree": 1.0,
   "n_calls": 67,
   "ci": {
    "level": [
     -12.025760683760682,
     18.493986842105254
    ],
    "spearman": [
     0.48336709492178176,
     0.8514568548894945
    ],
    "mmrv": [
     0.023021038790269566,
     0.10840000000000001
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9997139180494051,
   "rerun_spearman": 0.9143022472803871
  },
  "TIER real-replica": {
   "units": 5,
   "policies": 4,
   "cells": 20,
   "level": -1.0,
   "spearman": 0.6523920842049056,
   "mmrv": 0.04,
   "agree": 1.0,
   "n_calls": 6,
   "ci": {
    "level": [
     -4.0,
     2.0
    ],
    "spearman": [
     0.5267949192431123,
     0.875
    ],
    "mmrv": [
     0.013333333333333332,
     0.05333333333333333
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9950099206349204,
   "rerun_spearman": 0.6402336859296344
  },
  "Genie Sim 3.0 | Agibot G1, mirrored scenes": {
   "units": 4,
   "policies": 4,
   "cells": 16,
   "level": 3.25,
   "spearman": 0.9871708245126285,
   "mmrv": 0.004374999999999995,
   "agree": 1.0,
   "n_calls": 12,
   "ci": {
    "level": [
     1.5624999999999993,
     4.937500000000001
    ],
    "spearman": [
     0.9665063509461097,
     1.0
    ],
    "mmrv": [
     0.0,
     0.005833333333333327
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.8973293116464739
  },
  "MolmoBot | DROID Franka, related tasks (not mirrored)": {
   "units": 2,
   "policies": 7,
   "cells": 12,
   "level": 1.9166666666666659,
   "spearman": 0.875,
   "mmrv": 0.07614285714285712,
   "agree": 0.9444444444444444,
   "n_calls": 18,
   "ci": {
    "level": [
     -13.034285714285717,
     12.32307692307692
    ],
    "spearman": [
     0.6,
     1.0
    ],
    "mmrv": [
     0.0,
     0.13349999999999998
    ],
    "agree": [
     0.75,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.9220834770402363
  },
  "PolaRiS | DROID Franka, splat replica (sim co-fine-tuned)": {
   "units": 6,
   "policies": 5,
   "cells": 30,
   "level": 18.463056864048927,
   "spearman": 0.9124465724134826,
   "mmrv": 0.02987179487179488,
   "agree": 1.0,
   "n_calls": 31,
   "ci": {
    "level": [
     14.94217948761282,
     21.98393424048503
    ],
    "spearman": [
     0.8110042339640732,
     0.9333333333333332
    ],
    "mmrv": [
     0.0018518518518518571,
     0.036111111111111115
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9996428571428573,
   "rerun_spearman": 0.8710872164139709
  },
  "PolaRiS control | same real tasks, LIBERO score (not a twin)": {
   "units": 6,
   "policies": 5,
   "cells": 30,
   "level": -31.839188034188034,
   "spearman": 0.8457799057468159,
   "mmrv": 0.03917735042735043,
   "agree": 1.0,
   "n_calls": 17,
   "ci": {
    "level": [
     -38.91282051282051,
     -23.88336111111114
    ],
    "spearman": [
     0.5610042339640732,
     0.9166666666666666
    ],
    "mmrv": [
     0.0018518518518518571,
     0.04619391025641025
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9994922341696535,
   "rerun_spearman": 0.8724051010346295
  },
  "RoboDojo | ARX X5 real / ARX X5 sim": {
   "units": 2,
   "policies": 10,
   "cells": 20,
   "level": -2.4395000000000002,
   "spearman": 0.4297729398220962,
   "mmrv": 0.12355,
   "agree": 0.7878787878787878,
   "n_calls": 33,
   "ci": {
    "level": [
     -7.998525,
     2.606999999999998
    ],
    "spearman": [
     0.05484867948129099,
     0.7812143321460664
    ],
    "mmrv": [
     0.035910833333333336,
     0.14243499999999998
    ],
    "agree": [
     0.46153846153846156,
     1.0
    ]
   },
   "rerun_agree": 0.9991291918408415,
   "rerun_spearman": 0.8349434503389564
  },
  "RoboDojo | Piper X real / ARX X5 sim": {
   "units": 3,
   "policies": 10,
   "cells": 30,
   "level": -2.5543333333333336,
   "spearman": -0.0992074490411542,
   "mmrv": 0.035666666666666666,
   "agree": 0.0,
   "n_calls": 2,
   "ci": {
    "level": [
     -4.029083333333333,
     -1.284916666666667
    ],
    "spearman": [
     -0.408248290463863,
     0.3190896140869863
    ],
    "mmrv": [
     0.0044444444444444444,
     0.03666666666666667
    ],
    "agree": [
     0.0,
     0.0
    ]
   },
   "rerun_agree": 0.9823046536796537,
   "rerun_spearman": 0.5009568099169527
  },
  "RoboDojo | Piper real / ARX X5 sim": {
   "units": 2,
   "policies": 10,
   "cells": 20,
   "level": 1.8079999999999992,
   "spearman": 0.6004508409193556,
   "mmrv": 0.19335000000000002,
   "agree": 0.8780487804878049,
   "n_calls": 41,
   "ci": {
    "level": [
     -5.051875,
     8.605575
    ],
    "spearman": [
     0.2321428571428572,
     0.8491464808844296
    ],
    "mmrv": [
     0.05657,
     0.2322142857142857
    ],
    "agree": [
     0.5862068965517241,
     1.0
    ]
   },
   "rerun_agree": 0.9996938775510205,
   "rerun_spearman": 0.8856800066821986
  },
  "RoboLab-120 (v2) | Franka DROID, matched tasks (withdrawn after v2)": {
   "units": 1,
   "policies": 4,
   "cells": 4,
   "level": 9.700000000000001,
   "spearman": 0.7999999999999999,
   "mmrv": 0.14550000000000002,
   "agree": 1.0,
   "n_calls": 4,
   "ci": {
    "level": [
     -3.5749999999999993,
     22.975
    ],
    "spearman": [
     0.5,
     1.0
    ],
    "mmrv": [
     0.0,
     0.19400000000000003
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.9413291266553458
  },
  "SimplerEnv | Google Robot, variant aggregation": {
   "units": 7,
   "policies": 6,
   "cells": 40,
   "level": 18.265000000000004,
   "spearman": 0.667113235512143,
   "mmrv": 0.15820952380952383,
   "agree": 1.0,
   "n_calls": 50,
   "ci": {
    "level": [
     9.782333333333337,
     23.6875
    ],
    "spearman": [
     0.21428571428571427,
     0.8571428571428571
    ],
    "mmrv": [
     0.02904761904761906,
     0.1575066666666666
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9999107142857143,
   "rerun_spearman": 0.9493039529474603
  },
  "SimplerEnv | Google Robot, visual matching": {
   "units": 7,
   "policies": 6,
   "cells": 40,
   "level": 12.325,
   "spearman": 0.8835977665919683,
   "mmrv": 0.05223809523809525,
   "agree": 1.0,
   "n_calls": 60,
   "ci": {
    "level": [
     9.180255847953216,
     15.27857142857143
    ],
    "spearman": [
     0.6732050807568878,
     0.9285714285714286
    ],
    "mmrv": [
     0.010571428571428574,
     0.05133333333333335
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.9493866229150502
  },
  "SimplerEnv | WidowX, visual matching": {
   "units": 4,
   "policies": 3,
   "cells": 12,
   "level": 0.10833333333333399,
   "spearman": 0.875,
   "mmrv": 0.02783333333333333,
   "agree": 1.0,
   "n_calls": 4,
   "ci": {
    "level": [
     0.10833333333333399,
     0.10833333333333399
    ],
    "spearman": [
     0.875,
     0.875
    ],
    "mmrv": [
     0.02783333333333333,
     0.02783333333333333
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 1.0,
   "rerun_spearman": 0.944287376160107
  },
  "VLA-Replica | original lab vs rebuilt lab (both real)": {
   "units": 5,
   "policies": 4,
   "cells": 20,
   "level": -1.0,
   "spearman": 0.6523920842049056,
   "mmrv": 0.04,
   "agree": 1.0,
   "n_calls": 6,
   "ci": {
    "level": [
     -4.0,
     2.0
    ],
    "spearman": [
     0.5267949192431123,
     0.875
    ],
    "mmrv": [
     0.013333333333333332,
     0.05333333333333333
    ],
    "agree": [
     1.0,
     1.0
    ]
   },
   "rerun_agree": 0.9941289682539682,
   "rerun_spearman": 0.6263023644840636
  }
 },
 "direct_test_twins_without_sim_finetuning": {
  "units": 15,
  "policies": 11,
  "cells": 68,
  "level": 8.033823529411766,
  "spearman": 0.9089245109462861,
  "mmrv": 0.03296666666666667,
  "agree": 1.0,
  "n_calls": 76,
  "ci": {
   "level": [
    6.004761558063343,
    13.27391663048198
   ],
   "spearman": [
    0.7680621180045922,
    0.9545454545454546
   ],
   "mmrv": [
    0.005555555555555553,
    0.0444
   ],
   "agree": [
    1.0,
    1.0
   ]
  },
  "rerun_agree": 1.0,
  "rerun_spearman": 0.934580847073748
 },
 "index_sim_vs_real": {
  "ranked overall (1+ real, 1+ sim)": {
   "n": 37,
   "sim_real": [
    0.6795494961470065,
    0.4994552203461962,
    0.7915409948859626
   ],
   "sim_overall": [
    0.9071187260318869,
    0.8117283858080331,
    0.9514882521557916
   ]
  },
  "2+ sim and 2+ real benchmarks": {
   "n": 10,
   "sim_real": [
    0.8909090909090909,
    0.6397515527950312,
    0.9872611464968155
   ],
   "sim_overall": [
    0.9515151515151514,
    0.799874213836478,
    1.0
   ]
  },
  "exactly 1 sim or 1 real benchmark": {
   "n": 27,
   "sim_real": [
    0.606412213740458,
    0.357513912854252,
    0.7653839473387921
   ],
   "sim_overall": [
    0.8865822115200276,
    0.7255948337426532,
    0.9610014745249419
   ]
  },
  "robot policies only (no coding agents)": {
   "n": 29,
   "sim_real": [
    0.6109126786084997,
    0.3948274416543374,
    0.7571737221933793
   ],
   "sim_overall": [
    0.8912427706473999,
    0.7560333936179398,
    0.9539723585353052
   ]
  },
  "coding agents only": {
   "n": 8,
   "sim_real": [
    0.8571428571428572,
    0.5609756097560976,
    1.0
   ],
   "sim_overall": [
    0.9047619047619048,
    0.5555555555555556,
    1.0
   ]
  }
 },
 "aggregation_heldout": {
  "all": {
   "single benchmark (average)": [
    0.7086456608268689,
    0.5777236062287755,
    0.8202682215052869,
    149
   ],
   "composite, real only": [
    0.6923076923076923,
    0.3894444444444444,
    0.8421052631578947,
    78
   ],
   "composite, sim only": [
    0.723404255319149,
    0.571329365079365,
    0.832349896480331,
    141
   ],
   "composite, all": [
    0.6845637583892618,
    0.4998299319727892,
    0.808445012434374,
    149
   ],
   "diff": [
    -0.02408190243760713,
    -0.11591776895943563,
    0.03256395266439904,
    0.7466666666666667
   ]
  },
  "multi": {
   "single benchmark (average)": [
    0.6855738507524222,
    0.490113348716361,
    0.8133958089854197,
    84
   ],
   "composite, real only": [
    0.7142857142857143,
    0.3333333333333333,
    0.8488591487494515,
    63
   ],
   "composite, sim only": [
    0.6904761904761905,
    0.4499637681159421,
    0.8414694518232311,
    84
   ],
   "composite, all": [
    0.6785714285714286,
    0.39280888030888034,
    0.8243243243243243,
    84
   ],
   "diff": [
    -0.007002422180993609,
    -0.12474462365591397,
    0.06859382778679647,
    0.6156666666666667
   ]
  }
 },
 "aggregation_split_half": {
  "split_half_all": [
   0.6439468976239824,
   0.4839584929058613,
   0.7594458157008276,
   200
  ],
  "split_half_2plus": [
   0.7026629072681705,
   0.399010989010989,
   0.8846703296703297,
   200
  ],
  "single_pair_median": 0.5,
  "single_pair_weighted": 0.46807151574009725,
  "single_pairs": 143
 },
 "rank_swing": {
  "models_2plus": 62,
  "top3_somewhere": 22,
  "top3_and_bottom_half": 11,
  "median_swing_3plus": 43.442205942205945,
  "n_3plus": 32,
  "share_swing_over_50_3plus": 0.34375,
  "first_and_last": [
   "π0.5"
  ]
 },
 "shared_sim_mistakes": {
  "summary": {
   "matchups": 56,
   "sim_pairs": 385,
   "p_wrong": 0.18961038961038962,
   "p_wrong_given_other_wrong": 0.6421877988143048,
   "both_wrong_if_independent": 0.035277449822904365,
   "both_wrong_observed": 0.11948051948051948
  },
  "matchups": [
   {
    "real": "robodojo_real",
    "a": "X-VLA",
    "b": "π0.5",
    "sims_wrong": [
     "simpler_wx",
     "simpler_g",
     "calvin"
    ],
    "sims_right": [
     "robotwin",
     "molmospaces",
     "ebench",
     "libero_pro2"
    ],
    "favoured_by_sims": "X-VLA",
    "other_real_side_with_target": 2,
    "other_real_side_with_sims": 0
   },
   {
    "real": "robocurve",
    "a": "GPT-6 Astra",
    "b": "Gemini 4 Argon",
    "sims_wrong": [
     "robocoliseum",
     "efe"
    ],
    "sims_right": [],
    "favoured_by_sims": "GPT-6 Astra",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 0
   },
   {
    "real": "armnet",
    "a": "GR00T N1.7",
    "b": "MolmoAct2",
    "sims_wrong": [
     "robodojo_sim",
     "libero_pro2"
    ],
    "sims_right": [],
    "favoured_by_sims": "MolmoAct2",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 4
   },
   {
    "real": "armnet",
    "a": "MolmoAct2",
    "b": "π0",
    "sims_wrong": [
     "robodojo_sim",
     "molmospaces",
     "libero_pro2"
    ],
    "sims_right": [],
    "favoured_by_sims": "MolmoAct2",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 0
   },
   {
    "real": "armnet",
    "a": "MolmoAct2",
    "b": "π0.5",
    "sims_wrong": [
     "molmospaces",
     "libero_pro2"
    ],
    "sims_right": [],
    "favoured_by_sims": "MolmoAct2",
    "other_real_side_with_target": 2,
    "other_real_side_with_sims": 1
   },
   {
    "real": "vla_replica",
    "a": "MolmoAct2",
    "b": "π0.5",
    "sims_wrong": [
     "molmospaces",
     "libero_pro2"
    ],
    "sims_right": [],
    "favoured_by_sims": "MolmoAct2",
    "other_real_side_with_target": 2,
    "other_real_side_with_sims": 1
   },
   {
    "real": "vla_replica",
    "a": "SmolVLA",
    "b": "X-VLA",
    "sims_wrong": [
     "robodojo_sim",
     "libero"
    ],
    "sims_right": [],
    "favoured_by_sims": "X-VLA",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 0
   },
   {
    "real": "vla_replica",
    "a": "X-VLA",
    "b": "π0",
    "sims_wrong": [
     "robodojo_sim",
     "robotwin",
     "libero_pro2",
     "simpler_wx",
     "simpler_g",
     "calvin"
    ],
    "sims_right": [
     "ebench"
    ],
    "favoured_by_sims": "X-VLA",
    "other_real_side_with_target": 1,
    "other_real_side_with_sims": 0
   },
   {
    "real": "vla_replica",
    "a": "X-VLA",
    "b": "π0.5",
    "sims_wrong": [
     "simpler_wx",
     "simpler_g",
     "calvin"
    ],
    "sims_right": [
     "robotwin",
     "molmospaces",
     "ebench",
     "libero_pro2"
    ],
    "favoured_by_sims": "X-VLA",
    "other_real_side_with_target": 2,
    "other_real_side_with_sims": 0
   },
   {
    "real": "robochallenge",
    "a": "CogACT",
    "b": "π0",
    "sims_wrong": [
     "simpler_wx",
     "simpler_g"
    ],
    "sims_right": [
     "calvin"
    ],
    "favoured_by_sims": "CogACT",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 0
   },
   {
    "real": "robochallenge",
    "a": "OpenVLA-OFT",
    "b": "π0",
    "sims_wrong": [
     "libero_pro2",
     "libero_plus"
    ],
    "sims_right": [
     "robodojo_sim"
    ],
    "favoured_by_sims": "OpenVLA-OFT",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 0
   },
   {
    "real": "robochallenge",
    "a": "X-VLA",
    "b": "π0",
    "sims_wrong": [
     "robodojo_sim",
     "robotwin",
     "libero_pro2",
     "simpler_wx",
     "simpler_g",
     "calvin"
    ],
    "sims_right": [
     "ebench"
    ],
    "favoured_by_sims": "X-VLA",
    "other_real_side_with_target": 1,
    "other_real_side_with_sims": 0
   },
   {
    "real": "robochallenge",
    "a": "X-VLA",
    "b": "π0.5",
    "sims_wrong": [
     "simpler_wx",
     "simpler_g",
     "calvin"
    ],
    "sims_right": [
     "robotwin",
     "molmospaces",
     "ebench",
     "libero_pro2"
    ],
    "favoured_by_sims": "X-VLA",
    "other_real_side_with_target": 2,
    "other_real_side_with_sims": 0
   },
   {
    "real": "phail_droid",
    "a": "GR00T N1.7",
    "b": "π0.5",
    "sims_wrong": [
     "robodojo_sim",
     "robocoliseum",
     "libero_pro2"
    ],
    "sims_right": [],
    "favoured_by_sims": "π0.5",
    "other_real_side_with_target": 0,
    "other_real_side_with_sims": 5
   },
   {
    "real": "colosseum_ft",
    "a": "MolmoAct2",
    "b": "π0.5",
    "sims_wrong": [
     "molmospaces",
     "libero_pro2"
    ],
    "sims_right": [],
    "favoured_by_sims": "MolmoAct2",
    "other_real_side_with_target": 2,
    "other_real_side_with_sims": 1
   }
  ]
 },
 "matched_vs_unmatched": {
  "simpler": {
   "tasks": 7,
   "policies": 5,
   "spearman_matched": 0.853525633497271,
   "spearman_unmatched": 0.5941560604614005,
   "mmrv_matched": 0.03942857142857144,
   "mmrv_unmatched": 0.12962857142857143,
   "d_spearman_ci": [
    0.0,
    0.7142857142857142
   ]
  },
  "polaris": {
   "tasks": 6,
   "policies": 5,
   "spearman_matched": 0.9124465724134826,
   "spearman_unmatched": 0.8457799057468159,
   "mmrv_matched": 0.02987179487179488,
   "mmrv_unmatched": 0.03917735042735043,
   "d_spearman_ci": [
    0.0,
    0.1875
   ]
  }
 },
 "polaris_without_cofinetuning_r": 0.3,
 "model_families": {
  "cross_family": {
   "sim|WAM vs VLA": {
    "pairs": 244,
    "benchmarks": 8,
    "observed": 0.7300074618408607,
    "predicted": 0.6792515161745172,
    "gap": 0.05075594566634357,
    "lo": -0.01118514761925362,
    "hi": 0.10787229681257116,
    "p_le0": 0.08675,
    "lomo_min": 0.03887732265450043,
    "lomo_max": 0.06925593881024819,
    "lomo_min_model": "InternW0-Δ",
    "lobo": {
     "EBench": 0.05247086159144847,
     "LIBERO-Pro2.0": 0.04902166771372085,
     "MolmoSpaces": 0.05070825967609779,
     "RoboCasa365": 0.0597056661894428,
     "RoboColiseum": 0.05444929096782759,
     "RoboDojo (sim)": 0.06525870496350628,
     "RoboLab-120": 0.043665266672571146,
     "RoboTwin 2.0": 0.03123669054424297
    }
   },
   "sim|LLM agent vs VLA": {
    "pairs": 47,
    "benchmarks": 3,
    "observed": 0.9307793665253017,
    "predicted": 0.7776087218554981,
    "gap": 0.15317064466980332,
    "lo": 0.0846703934008094,
    "hi": 0.3063652297165654,
    "p_le0": 0.0012664640324214793,
    "lomo_min": 0.12585005927071716,
    "lomo_max": 0.22045236117672176,
    "lomo_min_model": "DeepSeek V4.1 Flash",
    "lobo": {
     "EBench": 0.1648050107850548,
     "RoboColiseum": 0.087208234183763,
     "RoboDojo (sim)": 0.17634162716659732
    }
   },
   "sim|LLM agent vs WAM": {
    "pairs": 20,
    "benchmarks": 3,
    "observed": 0.5698084712259773,
    "predicted": 0.5881071861749022,
    "gap": -0.01829871494892494,
    "lo": -0.1859478473078969,
    "hi": 0.4052619395167175,
    "p_le0": 0.5091288471570161,
    "lomo_min": -0.1057630804885494,
    "lomo_max": 0.18731941488324497,
    "lomo_min_model": "OpenWAM-α",
    "lobo": {
     "EBench": 0.08668275874934188,
     "RoboColiseum": -0.15409069138416406,
     "RoboDojo (sim)": -0.014084744814396734
    }
   },
   "real|WAM vs VLA": {
    "pairs": 38,
    "benchmarks": 4,
    "observed": 0.9931763289712814,
    "predicted": 0.7750553748038541,
    "gap": 0.21812095416742727,
    "lo": 0.12035535001687751,
    "hi": 0.34691932906083467,
    "p_le0": 0.0,
    "lomo_min": 0.17788473147036996,
    "lomo_max": 0.2452574089466763,
    "lomo_min_model": "DreamZero",
    "lobo": {
     "PhAIL (DROID tasks)": 0.2104041927110731,
     "RoboArena": 0.17788473147036996,
     "RoboDojo-RC": 0.20075898840324968,
     "RoboDojo (real)": 0.34187721974447405
    }
   },
   "real|LLM agent vs VLA": {
    "pairs": 20,
    "benchmarks": 2,
    "observed": 0.9961480484653802,
    "predicted": 0.8311820969820338,
    "gap": 0.1649659514833465,
    "lo": 0.09301646955601978,
    "hi": 0.2797320399659348,
    "p_le0": 0.0,
    "lomo_min": 0.13015565929678855,
    "lomo_max": 0.2105954705464122,
    "lomo_min_model": "MolmoAct2",
    "lobo": {
     "RoboColosseum (open)": 0.1811293571406849,
     "RoboDojo-RC": 0.1488025458260081
    }
   },
   "real|LLM agent vs WAM": {
    "pairs": 5,
    "benchmarks": 1,
    "observed": 0.0009508218657598596,
    "predicted": 0.8517351254741999,
    "gap": -0.8507843036084399,
    "lo": -0.9153166005017276,
    "hi": -0.7567309331890858,
    "p_le0": 1.0,
    "lomo_min": -0.9012797300683286,
    "lomo_max": -0.8296953598752973,
    "lomo_min_model": "Gemini 3.8 Flash",
    "lobo": {
     "RoboDojo-RC": NaN
    }
   },
   "by_bench": {
    "PhAIL (DROID tasks)": {
     "pairs": 8,
     "observed": 0.9985183397068087,
     "predicted": 0.7514595300780532,
     "wams": [
      "Cosmos3-Nano-Policy",
      "FLUX 3 Action"
     ]
    },
    "RoboArena": {
     "pairs": 7,
     "observed": 0.9646863008999854,
     "predicted": 0.568376360502733,
     "wams": [
      "DreamZero"
     ]
    },
    "RoboDojo-RC": {
     "pairs": 2,
     "observed": 1.0,
     "predicted": 0.469363662077376,
     "wams": [
      "Dream-1"
     ]
    },
    "RoboDojo (real)": {
     "pairs": 21,
     "observed": 0.999988079854968,
     "predicted": 0.882050769249912,
     "wams": [
      "InternW0-Δ",
      "MWA-Phoenix",
      "OpenWAM-α"
     ]
    }
   },
   "agents_within": {
    "RoboDojo-RC": {
     "n": 5,
     "rho": 0.49999999999999994,
     "models": [
      "Claude Opus 5.5",
      "GPT-6 Astra",
      "GPT-6.1 Sol",
      "Gemini 3.8 Flash",
      "Gemini 4 Argon"
     ]
    },
    "DataCenterBench": {
     "n": 7,
     "rho": 0.7500000000000002,
     "models": [
      "Claude Opus 5.5",
      "DeepSeek V4.1 Flash",
      "GLM 5.3 Flash",
      "GPT-6 Astra",
      "GPT-6.1 Sol",
      "Gemini 4 Argon",
      "Muse Wonder 1.0"
     ]
    },
    "AA-Robotics": {
     "n": 4,
     "rho": 0.19999999999999998,
     "models": [
      "Claude Opus 5.5",
      "GPT-6 Astra",
      "GPT-6.1 Sol",
      "Gemini 4 Argon"
     ]
    }
   }
  },
  "per_wam_real": [
   {
    "m": "Dream-1",
    "b": "RoboDojo-RC",
    "n": 2,
    "won": 2,
    "obs": 1.0,
    "pred": 0.469363662077376
   },
   {
    "m": "DreamZero",
    "b": "RoboArena",
    "n": 7,
    "won": 7,
    "obs": 0.9646863008999854,
    "pred": 0.568376360502733
   },
   {
    "m": "Cosmos3-Nano-Policy",
    "b": "PhAIL",
    "n": 4,
    "won": 4,
    "obs": 0.9998503800103646,
    "pred": 0.6977121034592101
   },
   {
    "m": "FLUX 3 Action",
    "b": "PhAIL",
    "n": 4,
    "won": 4,
    "obs": 0.9971862994032528,
    "pred": 0.8052069566968963
   },
   {
    "m": "OpenWAM-α",
    "b": "RoboDojo",
    "n": 7,
    "won": 7,
    "obs": 0.9999977046842695,
    "pred": 0.8671328344120495
   },
   {
    "m": "InternW0-Δ",
    "b": "RoboDojo",
    "n": 7,
    "won": 7,
    "obs": 0.9999995695884891,
    "pred": 0.8769977339048654
   },
   {
    "m": "MWA-Phoenix",
    "b": "RoboDojo",
    "n": 7,
    "won": 7,
    "obs": 0.9999669652921451,
    "pred": 0.9020217394328208
   }
  ],
  "residuals_by_family": {
   "sim|WAM": {
    "models": 12,
    "cells": 28,
    "benchmarks": 8,
    "mean": 0.039474438981473826,
    "lo": 0.0020812856840405065,
    "hi": 0.07624791716753443,
    "rank_pct": 0.5505739795918367,
    "p_above": 0.16095,
    "p_below": 0.8392,
    "share_positive": 0.5833333333333334
   },
   "sim|LLM agent": {
    "models": 4,
    "cells": 6,
    "benchmarks": 3,
    "mean": 0.11929766305631159,
    "lo": 0.054197452063039296,
    "hi": 0.18992038651892826,
    "rank_pct": 0.6757936507936507,
    "p_above": 0.06615,
    "p_below": 0.9344,
    "share_positive": 1.0
   },
   "sim|VLA": {
    "models": 21,
    "cells": 61,
    "benchmarks": 8,
    "mean": -0.02272454441265858,
    "lo": -0.06057203009232696,
    "hi": 0.01480415935960037,
    "rank_pct": 0.45949453551912567,
    "p_above": 0.9529,
    "p_below": 0.0471,
    "share_positive": 0.42857142857142855
   },
   "real|WAM": {
    "models": 7,
    "cells": 7,
    "benchmarks": 4,
    "mean": 0.26087207859305156,
    "lo": 0.12218048913816001,
    "hi": 0.41815081198389764,
    "rank_pct": 0.819047619047619,
    "p_above": 0.00185,
    "p_below": 0.99825,
    "share_positive": 0.8571428571428571
   },
   "real|LLM agent": {
    "models": 5,
    "cells": 7,
    "benchmarks": 2,
    "mean": 0.0017169725569465555,
    "lo": -0.0518430991802016,
    "hi": 0.07745145992331214,
    "rank_pct": 0.5850340136054422,
    "p_above": 0.1863,
    "p_below": 0.8206,
    "share_positive": 0.2
   },
   "real|VLA": {
    "models": 13,
    "cells": 25,
    "benchmarks": 5,
    "mean": -0.07102608763725021,
    "lo": -0.15605339934722232,
    "hi": 0.00878452577740013,
    "rank_pct": 0.3868571428571429,
    "p_above": 0.9992,
    "p_below": 0.0008,
    "share_positive": 0.3076923076923077
   }
  },
  "robustness": {
   "index yardstick (lam=30)": {
    "real": 0.21812095416742727,
    "real_ci": [
     0.12256751640877765,
     0.34705283692291894
    ],
    "sim": 0.050755945666343574,
    "sim_ci": [
     -0.01007564673122728,
     0.10967876947472574
    ],
    "diff": 0.1673650085010837,
    "diff_ci": [
     0.050170472877201666,
     0.30944017109432376
    ],
    "p_diff_le0": 0.0036666666666666666,
    "n_real": 38,
    "n_sim": 244
   },
   "coverage": {
    "wam": {
     "InternW0-Δ": 1,
     "OpenWAM-α": 2,
     "MWA-Phoenix": 1,
     "Dream-1": 1,
     "DreamZero": 2,
     "Cosmos3-Nano-Policy": 2,
     "FLUX 3 Action": 1
    },
    "vla": {
     "π0.5": 7,
     "GalaxeaVLA G0": 1,
     "Xiaomi-Robotics-0": 2,
     "X-VLA": 3,
     "GR00T N1.7": 2,
     "π0": 7,
     "StarVLA": 1,
     "MolmoAct2": 2,
     "π0-FAST": 2,
     "GalaxeaVLA G0.5": 1,
     "Phoenix": 1,
     "PaliGemma-binning": 1
    },
    "gap_low_cov_vla": 0.2407076779117312,
    "n_low": 14,
    "gap_highcov_vla": 0.1785238554141947,
    "n_high": 13
   },
   "weak prior (lam=3)": {
    "real": 0.1469834013353436,
    "real_ci": [
     0.0338305156952413,
     0.3254843130850171
    ],
    "sim": 0.013894706874178295,
    "sim_ci": [
     -0.02223992622722132,
     0.04856066958295504
    ],
    "diff": 0.1330886944611653,
    "diff_ci": [
     0.011642321358878315,
     0.3159331799439642
    ],
    "p_diff_le0": 0.026333333333333334,
    "n_real": 38,
    "n_sim": 244
   },
   "all sims incl. standard suites": {
    "real": 0.24700770417809528,
    "real_ci": [
     0.14359572510700838,
     0.3842968323244401
    ],
    "sim": 0.07545259578212797,
    "sim_ci": [
     0.01615239671996518,
     0.13393512482297823
    ],
    "diff": 0.17155510839596733,
    "diff_ci": [
     0.048417236048102254,
     0.31870028538986844
    ],
    "p_diff_le0": 0.007666666666666666,
    "n_real": 38,
    "n_sim": 261
   }
  },
  "recency": {
   "real": {
    "all": 0.21812095416742727,
    "n": 38,
    "earliest_wam": "2026-02-17",
    "recent_vla": 0.3356378124770226,
    "n_recent": 14,
    "recent_vlas": [
     "GR00T N1.7",
     "GalaxeaVLA G0.5",
     "MolmoAct2",
     "Phoenix"
    ],
    "intercept_same_date": 0.36481793265472173,
    "slope_per_year": -0.18341417105795918
   },
   "sim": {
    "all": 0.05016445914076397,
    "n": 237,
    "earliest_wam": "2026-02-17",
    "recent_vla": 0.004903337496840032,
    "n_recent": 63,
    "recent_vlas": [
     "Awomo-0.5I",
     "Awomo-0.5V",
     "EventVLA",
     "GR00T N1.7",
     "InternVLA-A1.5",
     "Meituan-Robotics-0",
     "MolmoAct2",
     "Xiaomi-Robotics-1"
    ],
    "intercept_same_date": -0.015160826731055115,
    "slope_per_year": 0.08132123010094794
   }
  },
  "index_placement": {
   "models": {
    "π0.5": {
     "type": "VLA",
     "sim": 0.7037037037037037,
     "real": 0.7407407407407407,
     "index": 0.7407407407407407
    },
    "GalaxeaVLA G0.5": {
     "type": "VLA",
     "sim": 0.8518518518518519,
     "real": 0.5185185185185185,
     "index": 0.7037037037037037
    },
    "MolmoAct2": {
     "type": "VLA",
     "sim": 0.7407407407407407,
     "real": 0.5555555555555556,
     "index": 0.6666666666666666
    },
    "Phoenix": {
     "type": "VLA",
     "sim": 0.7777777777777778,
     "real": 0.2222222222222222,
     "index": 0.6296296296296297
    },
    "GalaxeaVLA G0": {
     "type": "VLA",
     "sim": 0.48148148148148145,
     "real": 0.5925925925925926,
     "index": 0.5925925925925926
    },
    "InternVLA-A1": {
     "type": "VLA",
     "sim": 0.25925925925925924,
     "real": 0.7037037037037037,
     "index": 0.5555555555555556
    },
    "LingBot-VLA v2": {
     "type": "VLA",
     "sim": 0.6666666666666666,
     "real": 0.25925925925925924,
     "index": 0.5185185185185185
    },
    "Xiaomi-Robotics-0": {
     "type": "VLA",
     "sim": 0.4444444444444444,
     "real": 0.48148148148148145,
     "index": 0.48148148148148145
    },
    "GR00T N1.7": {
     "type": "VLA",
     "sim": 0.4074074074074074,
     "real": 0.4074074074074074,
     "index": 0.4444444444444444
    },
    "GR00T N1.6": {
     "type": "VLA",
     "sim": 0.2962962962962963,
     "real": 0.6296296296296297,
     "index": 0.4074074074074074
    },
    "X-VLA": {
     "type": "VLA",
     "sim": 0.5555555555555556,
     "real": 0.14814814814814814,
     "index": 0.37037037037037035
    },
    "π0-FAST": {
     "type": "VLA",
     "sim": 0.2222222222222222,
     "real": 0.6666666666666666,
     "index": 0.3333333333333333
    },
    "OpenVLA-OFT": {
     "type": "VLA",
     "sim": 0.37037037037037035,
     "real": 0.0,
     "index": 0.2962962962962963
    },
    "StarVLA": {
     "type": "VLA",
     "sim": 0.3333333333333333,
     "real": 0.2962962962962963,
     "index": 0.25925925925925924
    },
    "π0": {
     "type": "VLA",
     "sim": 0.14814814814814814,
     "real": 0.4444444444444444,
     "index": 0.2222222222222222
    },
    "DM0": {
     "type": "VLA",
     "sim": 0.1111111111111111,
     "real": 0.37037037037037035,
     "index": 0.18518518518518517
    },
    "Spirit v1.5": {
     "type": "VLA",
     "sim": 0.07407407407407407,
     "real": 0.3333333333333333,
     "index": 0.14814814814814814
    },
    "CogACT": {
     "type": "VLA",
     "sim": 0.5925925925925926,
     "real": 0.1111111111111111,
     "index": 0.1111111111111111
    },
    "RDT": {
     "type": "VLA",
     "sim": 0.0,
     "real": 0.18518518518518517,
     "index": 0.07407407407407407
    },
    "SmolVLA": {
     "type": "VLA",
     "sim": 0.18518518518518517,
     "real": 0.07407407407407407,
     "index": 0.037037037037037035
    },
    "PaliGemma-binning": {
     "type": "VLA",
     "sim": 0.037037037037037035,
     "real": 0.037037037037037035,
     "index": 0.0
    },
    "InternW0-Δ": {
     "type": "WAM",
     "sim": 0.9629629629629629,
     "real": 0.9629629629629629,
     "index": 1.0
    },
    "FLUX 3 Action": {
     "type": "WAM",
     "sim": 1.0,
     "real": 0.7777777777777778,
     "index": 0.9629629629629629
    },
    "MWA-Phoenix": {
     "type": "WAM",
     "sim": 0.9259259259259259,
     "real": 0.8148148148148148,
     "index": 0.9259259259259259
    },
    "OpenWAM-α": {
     "type": "WAM",
     "sim": 0.8888888888888888,
     "real": 0.8888888888888888,
     "index": 0.8888888888888888
    },
    "Cosmos3-Nano-Policy": {
     "type": "WAM",
     "sim": 0.8148148148148148,
     "real": 0.8518518518518519,
     "index": 0.8518518518518519
    },
    "Dream-1": {
     "type": "WAM",
     "sim": 0.5185185185185185,
     "real": 1.0,
     "index": 0.8148148148148148
    },
    "DreamZero": {
     "type": "WAM",
     "sim": 0.6296296296296297,
     "real": 0.9259259259259259,
     "index": 0.7777777777777778
    }
   },
   "n": 28
  },
  "matched_robodojo": {
   "VLA-WAM": {
    "n": 20,
    "agree": 20
   },
   "VLA-VLA": {
    "n": 23,
    "agree": 19
   }
  },
  "droid_same_robot": {
   "VLA-WAM": [
    10,
    12
   ],
   "VLA-VLA": [
    8,
    11
   ]
  },
  "direction_by_simulator": {
   "per_sim": {
    "robotwin": {
     "wam": [
      6,
      6
     ],
     "vla": [
      14,
      16
     ]
    },
    "robolab": {
     "wam": [
      5,
      7
     ],
     "vla": [
      10,
      12
     ]
    },
    "molmospaces": {
     "wam": [
      5,
      5
     ],
     "vla": [
      10,
      15
     ]
    },
    "robocoliseum": {
     "wam": [
      5,
      5
     ],
     "vla": [
      13,
      16
     ]
    },
    "ebench": {
     "wam": [
      6,
      6
     ],
     "vla": [
      10,
      10
     ]
    }
   },
   "total": {
    "wam": [
     27,
     29
    ],
    "vla_same_sims": [
     57,
     69
    ],
    "vla_all_sims": [
     164,
     241
    ]
   },
   "misses": [
    [
     "RoboLab-120",
     "RoboArena",
     "DreamZero",
     "Phoenix"
    ],
    [
     "RoboLab-120",
     "RoboArena",
     "DreamZero",
     "π0.5"
    ]
   ]
  },
  "calibration_real": {
   "vla_vs_vla": {
    "n": 118,
    "obs": 0.6446307493641213,
    "pred": 0.7094773719301812
   },
   "wam_favoured": {
    "n": 32,
    "obs": 0.9995721692184929,
    "pred": 0.8486083052106802
   }
  },
  "wam_outside_coverage": [
   {
    "m": "Cosmos3-Nano-Policy",
    "real": "PhAIL (DROID tasks)",
    "pred": 0.6977121034592101,
    "obs": 0.9998503800103646,
    "sims": [
     {
      "sim": "RoboLab-120",
      "rank": 4,
      "of": 12
     },
     {
      "sim": "MolmoSpaces",
      "rank": 4,
      "of": 10
     }
    ]
   },
   {
    "m": "Dream-1",
    "real": "RoboDojo-RC",
    "pred": 0.469363662077376,
    "obs": 1.0,
    "sims": [
     {
      "sim": "RoboColiseum",
      "rank": 14,
      "of": 19
     }
    ]
   },
   {
    "m": "DreamZero",
    "real": "RoboArena",
    "pred": 0.568376360502733,
    "obs": 0.9646863008999854,
    "sims": [
     {
      "sim": "RoboLab-120",
      "rank": 7,
      "of": 12
     },
     {
      "sim": "MolmoSpaces",
      "rank": 9,
      "of": 10
     }
    ]
   },
   {
    "m": "FLUX 3 Action",
    "real": "PhAIL (DROID tasks)",
    "pred": 0.8052069566968965,
    "obs": 0.9971862994032528,
    "sims": [
     {
      "sim": "RoboLab-120",
      "rank": 2,
      "of": 12
     }
    ]
   },
   {
    "m": "InternW0-Δ",
    "real": "RoboDojo (real)",
    "pred": 0.8769977339048654,
    "obs": 0.9999995695884891,
    "sims": [
     {
      "sim": "EBench",
      "rank": 3,
      "of": 11
     }
    ]
   },
   {
    "m": "MWA-Phoenix",
    "real": "RoboDojo (real)",
    "pred": 0.9020217394328208,
    "obs": 0.9999669652921451,
    "sims": [
     {
      "sim": "RoboTwin 2.0",
      "rank": 3,
      "of": 21
     }
    ]
   },
   {
    "m": "OpenWAM-α",
    "real": "RoboDojo (real)",
    "pred": 0.8671328344120494,
    "obs": 0.9999977046842696,
    "sims": [
     {
      "sim": "RoboColiseum",
      "rank": 10,
      "of": 19
     },
     {
      "sim": "EBench",
      "rank": 2,
      "of": 11
     }
    ]
   }
  ]
 }
}