{
  "haiku-main": {
    "model": "haiku",
    "note": "自動メモリOFF。本命の結果",
    "conditions": {
      "A": {
        "final_heldout": [124, 117, 152, 121, 116],
        "mean_final_heldout": 126,
        "trials": [
          {
            "trial": "A-k1",
            "sessions": [
              {
                "heldout": 117,
                "visible": 101,
                "turns": 16,
                "cost_usd": 0.2072,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 120,
                "visible": 107,
                "turns": 16,
                "cost_usd": 0.0993,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 120,
                "visible": 107,
                "turns": 16,
                "cost_usd": 0.0901,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 124,
                "visible": 112,
                "turns": 16,
                "cost_usd": 0.4898,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 124,
                "visible": 112,
                "turns": 16,
                "cost_usd": 0.1497,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-k2",
            "sessions": [
              {
                "heldout": 124,
                "visible": 127,
                "turns": 16,
                "cost_usd": 0.3463,
                "edits": 7,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 129,
                "visible": 128,
                "turns": 16,
                "cost_usd": 0.1201,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 125,
                "visible": 119,
                "turns": 16,
                "cost_usd": 0.1819,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 117,
                "visible": 111,
                "turns": 16,
                "cost_usd": 0.1084,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 117,
                "visible": 111,
                "turns": 16,
                "cost_usd": 0.1103,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-k3",
            "sessions": [
              {
                "heldout": 152,
                "visible": 138,
                "turns": 16,
                "cost_usd": 0.336,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 152,
                "visible": 138,
                "turns": 16,
                "cost_usd": 0.1002,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 155,
                "visible": 141,
                "turns": 16,
                "cost_usd": 0.1497,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 152,
                "visible": 141,
                "turns": 16,
                "cost_usd": 0.1031,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 152,
                "visible": 141,
                "turns": 16,
                "cost_usd": 0.0947,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-k4",
            "sessions": [
              {
                "heldout": 1,
                "visible": 0,
                "turns": 16,
                "cost_usd": 0.2109,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 112,
                "visible": 87,
                "turns": 16,
                "cost_usd": 0.1157,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 119,
                "visible": 94,
                "turns": 16,
                "cost_usd": 0.1241,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 119,
                "visible": 94,
                "turns": 16,
                "cost_usd": 0.1224,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 121,
                "visible": 96,
                "turns": 16,
                "cost_usd": 0.112,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-k5",
            "sessions": [
              {
                "heldout": 102,
                "visible": 81,
                "turns": 16,
                "cost_usd": 0.258,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 95,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1316,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 95,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1047,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 110,
                "visible": 91,
                "turns": 16,
                "cost_usd": 0.145,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 116,
                "visible": 94,
                "turns": 16,
                "cost_usd": 0.133,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "B": {
        "final_heldout": [87, 96, 91, 74, 109],
        "mean_final_heldout": 91.4,
        "trials": [
          {
            "trial": "B-k1",
            "sessions": [
              {
                "heldout": 86,
                "visible": 74,
                "turns": 16,
                "cost_usd": 0.1791,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 86,
                "visible": 75,
                "turns": 16,
                "cost_usd": 0.1,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 86,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1708,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 87,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1157,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 87,
                "visible": 80,
                "turns": 16,
                "cost_usd": 0.126,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "B-k2",
            "sessions": [
              {
                "heldout": 96,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.2191,
                "edits": 7,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 96,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.0981,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 96,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1044,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 96,
                "visible": 80,
                "turns": 16,
                "cost_usd": 0.1524,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 96,
                "visible": 80,
                "turns": 16,
                "cost_usd": 0.1127,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "B-k3",
            "sessions": [
              {
                "heldout": 43,
                "visible": 40,
                "turns": 16,
                "cost_usd": 0.1665,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 44,
                "visible": 44,
                "turns": 16,
                "cost_usd": 0.1238,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 44,
                "visible": 45,
                "turns": 16,
                "cost_usd": 0.1909,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 91,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.1367,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 91,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.1279,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "B-k4",
            "sessions": [
              {
                "heldout": 69,
                "visible": 49,
                "turns": 16,
                "cost_usd": 0.1636,
                "edits": 4,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 74,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.1063,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 73,
                "visible": 64,
                "turns": 16,
                "cost_usd": 0.1064,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 74,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.1021,
                "edits": 2,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 74,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.092,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "B-k5",
            "sessions": [
              {
                "heldout": 96,
                "visible": 76,
                "turns": 16,
                "cost_usd": 0.1912,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 105,
                "visible": 81,
                "turns": 16,
                "cost_usd": 0.2549,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 110,
                "visible": 96,
                "turns": 16,
                "cost_usd": 0.1839,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 108,
                "visible": 85,
                "turns": 16,
                "cost_usd": 0.0972,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 109,
                "visible": 95,
                "turns": 16,
                "cost_usd": 0.1448,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "C": {
        "final_heldout": [146, 99, 79, 144, 111],
        "mean_final_heldout": 115.8,
        "trials": [
          {
            "trial": "C-k1",
            "sessions": [
              {
                "heldout": 127,
                "visible": 118,
                "turns": 16,
                "cost_usd": 0.217,
                "edits": 8,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 130,
                "visible": 122,
                "turns": 16,
                "cost_usd": 0.1117,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 132,
                "visible": 126,
                "turns": 16,
                "cost_usd": 0.1199,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 141,
                "visible": 137,
                "turns": 16,
                "cost_usd": 0.1879,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 146,
                "visible": 143,
                "turns": 16,
                "cost_usd": 0.1556,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "C-k2",
            "sessions": [
              {
                "heldout": 82,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.3654,
                "edits": 9,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 83,
                "visible": 72,
                "turns": 16,
                "cost_usd": 0.1347,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 84,
                "turns": 16,
                "cost_usd": 0.1354,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 99,
                "visible": 83,
                "turns": 16,
                "cost_usd": 0.1356,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 99,
                "visible": 83,
                "turns": 16,
                "cost_usd": 0.1844,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "C-k3",
            "sessions": [
              {
                "heldout": 75,
                "visible": 56,
                "turns": 16,
                "cost_usd": 0.2268,
                "edits": 7,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 75,
                "visible": 62,
                "turns": 16,
                "cost_usd": 0.1366,
                "edits": 5,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 75,
                "visible": 62,
                "turns": 16,
                "cost_usd": 0.1077,
                "edits": 2,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 81,
                "visible": 71,
                "turns": 16,
                "cost_usd": 0.1498,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 79,
                "visible": 71,
                "turns": 16,
                "cost_usd": 0.1164,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "C-k4",
            "sessions": [
              {
                "heldout": 127,
                "visible": 110,
                "turns": 16,
                "cost_usd": 0.1603,
                "edits": 5,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 109,
                "visible": 93,
                "turns": 16,
                "cost_usd": 0.3397,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 134,
                "visible": 119,
                "turns": 16,
                "cost_usd": 0.2139,
                "edits": 6,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 138,
                "visible": 124,
                "turns": 16,
                "cost_usd": 0.155,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 144,
                "visible": 127,
                "turns": 16,
                "cost_usd": 0.144,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "C-k5",
            "sessions": [
              {
                "heldout": 75,
                "visible": 57,
                "turns": 16,
                "cost_usd": 0.1195,
                "edits": 5,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 87,
                "visible": 75,
                "turns": 16,
                "cost_usd": 0.1363,
                "edits": 2,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 104,
                "visible": 86,
                "turns": 16,
                "cost_usd": 0.1412,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 104,
                "visible": 94,
                "turns": 16,
                "cost_usd": 0.1543,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 111,
                "visible": 99,
                "turns": 16,
                "cost_usd": 0.1482,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "D": {
        "final_heldout": [82, 126, 122, 109, 99],
        "mean_final_heldout": 107.6,
        "trials": [
          {
            "trial": "D-k1",
            "sessions": [
              {
                "heldout": 86,
                "visible": 63,
                "turns": 16,
                "cost_usd": 0.195,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 86,
                "visible": 64,
                "turns": 16,
                "cost_usd": 0.1068,
                "edits": 2,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 87,
                "visible": 68,
                "turns": 16,
                "cost_usd": 0.1334,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 88,
                "visible": 68,
                "turns": 16,
                "cost_usd": 0.122,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 82,
                "visible": 65,
                "turns": 16,
                "cost_usd": 0.1976,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "D-k2",
            "sessions": [
              {
                "heldout": 121,
                "visible": 115,
                "turns": 16,
                "cost_usd": 0.1805,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 120,
                "visible": 115,
                "turns": 16,
                "cost_usd": 0.1062,
                "edits": 2,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 123,
                "visible": 117,
                "turns": 16,
                "cost_usd": 0.1249,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 128,
                "visible": 124,
                "turns": 16,
                "cost_usd": 0.1248,
                "edits": 4,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 126,
                "visible": 116,
                "turns": 16,
                "cost_usd": 0.1576,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "D-k3",
            "sessions": [
              {
                "heldout": 96,
                "visible": 69,
                "turns": 16,
                "cost_usd": 0.201,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 97,
                "visible": 75,
                "turns": 16,
                "cost_usd": 0.1032,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 107,
                "visible": 88,
                "turns": 16,
                "cost_usd": 0.1666,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 112,
                "visible": 91,
                "turns": 16,
                "cost_usd": 0.0973,
                "edits": 3,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 122,
                "visible": 88,
                "turns": 16,
                "cost_usd": 0.1633,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "D-k4",
            "sessions": [
              {
                "heldout": 99,
                "visible": 76,
                "turns": 16,
                "cost_usd": 0.1403,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 107,
                "visible": 87,
                "turns": 16,
                "cost_usd": 0.1251,
                "edits": 5,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 109,
                "visible": 90,
                "turns": 16,
                "cost_usd": 0.1073,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 109,
                "visible": 90,
                "turns": 16,
                "cost_usd": 0.0853,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 109,
                "visible": 92,
                "turns": 16,
                "cost_usd": 0.1109,
                "edits": 6,
                "progress_updated": true,
                "committed": false
              }
            ]
          },
          {
            "trial": "D-k5",
            "sessions": [
              {
                "heldout": 88,
                "visible": 76,
                "turns": 16,
                "cost_usd": 0.1678,
                "edits": 5,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 90,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1945,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 90,
                "visible": 79,
                "turns": 16,
                "cost_usd": 0.1786,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 97,
                "visible": 91,
                "turns": 16,
                "cost_usd": 0.2141,
                "edits": 3,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 99,
                "visible": 93,
                "turns": 16,
                "cost_usd": 0.1087,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "E": {
        "final_heldout": [87, 63, 67, 61, 58],
        "mean_final_heldout": 67.2,
        "trials": [
          {
            "trial": "E-k1",
            "sessions": [
              {
                "heldout": 42,
                "visible": 28,
                "turns": 16,
                "cost_usd": 0.1991,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 84,
                "visible": 73,
                "turns": 16,
                "cost_usd": 0.1725,
                "edits": 4,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 87,
                "visible": 78,
                "turns": 16,
                "cost_usd": 0.2165,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 5,
                "visible": 3,
                "turns": 16,
                "cost_usd": 0.1473,
                "edits": 4,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 87,
                "visible": 88,
                "turns": 16,
                "cost_usd": 0.0836,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "E-k2",
            "sessions": [
              {
                "heldout": 70,
                "visible": 54,
                "turns": 16,
                "cost_usd": 0.1604,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 70,
                "visible": 54,
                "turns": 16,
                "cost_usd": 0.1593,
                "edits": 2,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 62,
                "visible": 49,
                "turns": 16,
                "cost_usd": 0.1665,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 63,
                "visible": 50,
                "turns": 16,
                "cost_usd": 0.113,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 63,
                "visible": 50,
                "turns": 16,
                "cost_usd": 0.1105,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "E-k3",
            "sessions": [
              {
                "heldout": 56,
                "visible": 56,
                "turns": 16,
                "cost_usd": 0.1715,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 65,
                "visible": 64,
                "turns": 16,
                "cost_usd": 0.1093,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 65,
                "visible": 64,
                "turns": 16,
                "cost_usd": 0.0972,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 67,
                "visible": 63,
                "turns": 16,
                "cost_usd": 0.1804,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 67,
                "visible": 63,
                "turns": 16,
                "cost_usd": 0.1206,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "E-k4",
            "sessions": [
              {
                "heldout": 52,
                "visible": 43,
                "turns": 16,
                "cost_usd": 0.1661,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 59,
                "visible": 61,
                "turns": 16,
                "cost_usd": 0.0961,
                "edits": 2,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 59,
                "visible": 65,
                "turns": 16,
                "cost_usd": 0.1729,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 61,
                "visible": 67,
                "turns": 16,
                "cost_usd": 0.161,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 61,
                "visible": 67,
                "turns": 16,
                "cost_usd": 0.1628,
                "edits": 2,
                "progress_updated": true,
                "committed": true
              }
            ]
          },
          {
            "trial": "E-k5",
            "sessions": [
              {
                "heldout": 57,
                "visible": 51,
                "turns": 16,
                "cost_usd": 0.1454,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 56,
                "visible": 51,
                "turns": 16,
                "cost_usd": 0.1021,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 56,
                "visible": 51,
                "turns": 16,
                "cost_usd": 0.087,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 58,
                "visible": 58,
                "turns": 16,
                "cost_usd": 0.1118,
                "edits": 2,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 58,
                "visible": 60,
                "turns": 16,
                "cost_usd": 0.1407,
                "edits": 3,
                "progress_updated": true,
                "committed": true
              }
            ]
          }
        ]
      }
    }
  },
  "haiku-first": {
    "model": "haiku",
    "note": "自動メモリON（A-h3だけがメモリに書いた）。予備ラウンド",
    "conditions": {
      "A": {
        "final_heldout": [98, 118, 98],
        "mean_final_heldout": 104.7,
        "trials": [
          {
            "trial": "A-h1",
            "sessions": [
              {
                "heldout": 97,
                "visible": 78,
                "turns": 16,
                "cost_usd": 0.2152,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 78,
                "turns": 16,
                "cost_usd": 0.1404,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 78,
                "turns": 16,
                "cost_usd": 0.0746,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 78,
                "turns": 16,
                "cost_usd": 0.1145,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 78,
                "turns": 16,
                "cost_usd": 0.0725,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-h2",
            "sessions": [
              {
                "heldout": 98,
                "visible": 87,
                "turns": 16,
                "cost_usd": 0.2504,
                "edits": 7,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 96,
                "visible": 90,
                "turns": 16,
                "cost_usd": 0.1491,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 108,
                "visible": 108,
                "turns": 16,
                "cost_usd": 0.1385,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 118,
                "visible": 113,
                "turns": 16,
                "cost_usd": 0.1139,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 118,
                "visible": 113,
                "turns": 16,
                "cost_usd": 0.1149,
                "edits": 8,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-h3",
            "sessions": [
              {
                "heldout": 94,
                "visible": 90,
                "turns": 16,
                "cost_usd": 0.328,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 94,
                "visible": 90,
                "turns": 16,
                "cost_usd": 0.1194,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 96,
                "turns": 16,
                "cost_usd": 0.1468,
                "edits": 7,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 96,
                "turns": 16,
                "cost_usd": 0.0772,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 98,
                "visible": 96,
                "turns": 16,
                "cost_usd": 0.0839,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "B": {
        "final_heldout": [100, 63, 67],
        "mean_final_heldout": 76.7,
        "trials": [
          {
            "trial": "B-h1",
            "sessions": [
              {
                "heldout": 110,
                "visible": 99,
                "turns": 16,
                "cost_usd": 0.2521,
                "edits": 5,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 112,
                "visible": 101,
                "turns": 16,
                "cost_usd": 0.1202,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 112,
                "visible": 101,
                "turns": 16,
                "cost_usd": 0.1218,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 112,
                "visible": 102,
                "turns": 16,
                "cost_usd": 0.1286,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 100,
                "visible": 98,
                "turns": 16,
                "cost_usd": 0.2051,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "B-h2",
            "sessions": [
              {
                "heldout": 9,
                "visible": 5,
                "turns": 16,
                "cost_usd": 0.1865,
                "edits": 5,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 10,
                "visible": 5,
                "turns": 16,
                "cost_usd": 0.182,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 71,
                "visible": 54,
                "turns": 16,
                "cost_usd": 0.1904,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 71,
                "visible": 54,
                "turns": 16,
                "cost_usd": 0.0917,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 63,
                "visible": 46,
                "turns": 16,
                "cost_usd": 0.2429,
                "edits": 8,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "B-h3",
            "sessions": [
              {
                "heldout": 68,
                "visible": 56,
                "turns": 16,
                "cost_usd": 0.1871,
                "edits": 6,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 67,
                "visible": 58,
                "turns": 16,
                "cost_usd": 0.1363,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 67,
                "visible": 61,
                "turns": 16,
                "cost_usd": 0.128,
                "edits": 2,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 64,
                "visible": 57,
                "turns": 16,
                "cost_usd": 0.1842,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 67,
                "visible": 66,
                "turns": 16,
                "cost_usd": 0.177,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "C": {
        "final_heldout": [148, 141, 105],
        "mean_final_heldout": 131.3,
        "trials": [
          {
            "trial": "C-h1",
            "sessions": [
              {
                "heldout": 112,
                "visible": 98,
                "turns": 16,
                "cost_usd": 0.1908,
                "edits": 5,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 127,
                "visible": 110,
                "turns": 16,
                "cost_usd": 0.1245,
                "edits": 8,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 129,
                "visible": 114,
                "turns": 16,
                "cost_usd": 0.1385,
                "edits": 9,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 134,
                "visible": 118,
                "turns": 16,
                "cost_usd": 0.1197,
                "edits": 1,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 148,
                "visible": 128,
                "turns": 16,
                "cost_usd": 0.165,
                "edits": 3,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "C-h2",
            "sessions": [
              {
                "heldout": 108,
                "visible": 86,
                "turns": 16,
                "cost_usd": 0.1706,
                "edits": 4,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 115,
                "visible": 96,
                "turns": 16,
                "cost_usd": 0.142,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 128,
                "visible": 107,
                "turns": 16,
                "cost_usd": 0.1573,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 132,
                "visible": 108,
                "turns": 16,
                "cost_usd": 0.1378,
                "edits": 5,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 141,
                "visible": 115,
                "turns": 16,
                "cost_usd": 0.1486,
                "edits": 4,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "C-h3",
            "sessions": [
              {
                "heldout": 97,
                "visible": 92,
                "turns": 16,
                "cost_usd": 0.3527,
                "edits": 5,
                "progress_updated": true,
                "committed": false
              },
              {
                "heldout": 105,
                "visible": 97,
                "turns": 16,
                "cost_usd": 0.199,
                "edits": 6,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 106,
                "visible": 88,
                "turns": 16,
                "cost_usd": 0.2101,
                "edits": 4,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 104,
                "visible": 88,
                "turns": 16,
                "cost_usd": 0.1143,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 105,
                "visible": 88,
                "turns": 16,
                "cost_usd": 0.1206,
                "edits": 3,
                "progress_updated": true,
                "committed": false
              }
            ]
          }
        ]
      }
    }
  },
  "sonnet": {
    "model": "sonnet",
    "note": "自動メモリON。1セッション目で全問合格",
    "conditions": {
      "A": {
        "final_heldout": [326, 326, 326],
        "mean_final_heldout": 326,
        "trials": [
          {
            "trial": "A-s1",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 9,
                "cost_usd": 0.5683,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 9,
                "cost_usd": 0.0909,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 7,
                "cost_usd": 0.0624,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-s2",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 7,
                "cost_usd": 0.5338,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 5,
                "cost_usd": 0.0632,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 9,
                "cost_usd": 0.0893,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          },
          {
            "trial": "A-s3",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 12,
                "cost_usd": 0.5987,
                "edits": 2,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 6,
                "cost_usd": 0.0672,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 8,
                "cost_usd": 0.0834,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              }
            ]
          }
        ]
      },
      "B": {
        "final_heldout": [326, 326, 326],
        "mean_final_heldout": 326,
        "trials": [
          {
            "trial": "B-s1",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 10,
                "cost_usd": 0.5987,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 10,
                "cost_usd": 0.1018,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 4,
                "cost_usd": 0.0473,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              }
            ]
          },
          {
            "trial": "B-s2",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 10,
                "cost_usd": 0.644,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 8,
                "cost_usd": 0.0932,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 6,
                "cost_usd": 0.0556,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              }
            ]
          },
          {
            "trial": "B-s3",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 14,
                "cost_usd": 0.6951,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": null,
                "cost_usd": 0,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 5,
                "cost_usd": 0.0605,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              }
            ]
          }
        ]
      },
      "C": {
        "final_heldout": [326, 326, 326],
        "mean_final_heldout": 326,
        "trials": [
          {
            "trial": "C-s1",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 8,
                "cost_usd": 0.5572,
                "edits": 2,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 8,
                "cost_usd": 0.1266,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 6,
                "cost_usd": 0.0697,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              }
            ]
          },
          {
            "trial": "C-s2",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 11,
                "cost_usd": 0.6001,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": null,
                "cost_usd": 0,
                "edits": 0,
                "progress_updated": false,
                "committed": false
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 5,
                "cost_usd": 0.046,
                "edits": 0,
                "progress_updated": false,
                "committed": true
              }
            ]
          },
          {
            "trial": "C-s3",
            "sessions": [
              {
                "heldout": 326,
                "visible": 326,
                "turns": 9,
                "cost_usd": 0.5801,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 8,
                "cost_usd": 0.0847,
                "edits": 0,
                "progress_updated": false,
                "committed": true
              },
              {
                "heldout": 326,
                "visible": 326,
                "turns": 6,
                "cost_usd": 0.0532,
                "edits": 0,
                "progress_updated": true,
                "committed": true
              }
            ]
          }
        ]
      }
    }
  }
}