{
  "name": "Grace 0.3.7 vs Claude Code + caveman + context-mode, 2 October 2026 (same hours)",
  "date": "2026-10-02",
  "license": "Published by Grace for anyone to check. Please cite the source.",
  "setup": {
    "plan": "Claude Max 20x on every account",
    "baseline": "Claude Code with the token-saving plugins caveman and context-mode",
    "baselinePlugins": [
      "caveman",
      "context-mode"
    ],
    "parallelSessions": 3,
    "tasks": "30 new tasks, 6 per level (easy, medium, hard, extreme, mixed), none of them used in earlier tests of Grace",
    "quality": "share of the secret acceptance criteria met, judged per task (0–1); met counts 1, partly met 0.5, not met 0",
    "claudeCostUsd": "Claude's cost for the run at Anthropic API list prices, as Claude Code reports it (USD). The accounts are subscriptions and were not billed per token; the figure only sizes each run's Claude work.",
    "claudeOutputTokens": "tokens Claude wrote during the run (all of the run's Claude replies)",
    "fiveHourLimit": "the account's own usage display (5-hour and weekly limit), read every 5 minutes; shown in whole percentage points",
    "folders": "every run started in an empty folder; some tasks give starter code in the task text"
  },
  "runs": {
    "phase4_037": {
      "date": "2026-10-02",
      "version": "0.3.7",
      "simultaneous": true,
      "start": "2026-10-02T09:02:47Z",
      "end": "2026-10-02T11:36:21Z",
      "fiveHourLimitUsed": {
        "graceRead": 0.23,
        "graceOtherUsagePoints": 1.01,
        "graceAdjusted": 0.2199,
        "alone": 0.32
      },
      "weeklyLimitUsed": {
        "grace": 0.04,
        "alone": 0.05
      },
      "what": "Grace 0.3.7 (account A) and the baseline (account B) on the same 30 tasks, in the same order, at the same time",
      "summary": {
        "tasks": 30,
        "scoredBoth": 29,
        "qualityScoredBoth": {
          "grace": 0.9489,
          "alone": 0.937
        },
        "all": {
          "grace": {
            "claudeCostUsd": 57.3,
            "claudeOutputTokens": 2254983,
            "durationSeconds": 18909
          },
          "alone": {
            "claudeCostUsd": 96.77,
            "claudeOutputTokens": 3009584,
            "durationSeconds": 26505
          }
        },
        "graceFaster": 28,
        "graceCostAtMostHalf": [
          "X016",
          "X018",
          "G040",
          "X023",
          "E019"
        ]
      },
      "readings": {
        "grace": [
          {
            "minute": 0,
            "fiveHour": 0.02,
            "weekly": 0
          },
          {
            "minute": 5,
            "fiveHour": 0.04,
            "weekly": 0.01
          },
          {
            "minute": 10,
            "fiveHour": 0.05,
            "weekly": 0.01
          },
          {
            "minute": 15,
            "fiveHour": 0.06,
            "weekly": 0.01
          },
          {
            "minute": 20,
            "fiveHour": 0.07,
            "weekly": 0.01
          },
          {
            "minute": 26,
            "fiveHour": 0.08,
            "weekly": 0.01
          },
          {
            "minute": 31,
            "fiveHour": 0.09,
            "weekly": 0.02
          },
          {
            "minute": 36,
            "fiveHour": 0.11,
            "weekly": 0.02
          },
          {
            "minute": 41,
            "fiveHour": 0.12,
            "weekly": 0.02
          },
          {
            "minute": 46,
            "fiveHour": 0.13,
            "weekly": 0.02
          },
          {
            "minute": 51,
            "fiveHour": 0.14,
            "weekly": 0.02
          },
          {
            "minute": 56,
            "fiveHour": 0.15,
            "weekly": 0.02
          },
          {
            "minute": 61,
            "fiveHour": 0.16,
            "weekly": 0.03
          },
          {
            "minute": 66,
            "fiveHour": 0.17,
            "weekly": 0.03
          },
          {
            "minute": 71,
            "fiveHour": 0.18,
            "weekly": 0.03
          },
          {
            "minute": 76,
            "fiveHour": 0.19,
            "weekly": 0.03
          },
          {
            "minute": 82,
            "fiveHour": 0.2,
            "weekly": 0.03
          },
          {
            "minute": 87,
            "fiveHour": 0.21,
            "weekly": 0.04
          },
          {
            "minute": 92,
            "fiveHour": 0.22,
            "weekly": 0.04
          },
          {
            "minute": 97,
            "fiveHour": 0.23,
            "weekly": 0.04
          },
          {
            "minute": 102,
            "fiveHour": 0.24,
            "weekly": 0.04
          },
          {
            "minute": 107,
            "fiveHour": 0.24,
            "weekly": 0.04
          },
          {
            "minute": 112,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 117,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 122,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 127,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 133,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 138,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 143,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 148,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 153,
            "fiveHour": 0.25,
            "weekly": 0.04
          },
          {
            "minute": 154,
            "fiveHour": 0.25,
            "weekly": 0.04
          }
        ],
        "alone": [
          {
            "minute": 0,
            "fiveHour": 0.01,
            "weekly": 0.36
          },
          {
            "minute": 5,
            "fiveHour": 0.02,
            "weekly": 0.36
          },
          {
            "minute": 10,
            "fiveHour": 0.02,
            "weekly": 0.36
          },
          {
            "minute": 15,
            "fiveHour": 0.04,
            "weekly": 0.36
          },
          {
            "minute": 20,
            "fiveHour": 0.04,
            "weekly": 0.37
          },
          {
            "minute": 26,
            "fiveHour": 0.06,
            "weekly": 0.37
          },
          {
            "minute": 31,
            "fiveHour": 0.07,
            "weekly": 0.37
          },
          {
            "minute": 36,
            "fiveHour": 0.08,
            "weekly": 0.37
          },
          {
            "minute": 41,
            "fiveHour": 0.09,
            "weekly": 0.37
          },
          {
            "minute": 46,
            "fiveHour": 0.11,
            "weekly": 0.38
          },
          {
            "minute": 51,
            "fiveHour": 0.12,
            "weekly": 0.38
          },
          {
            "minute": 56,
            "fiveHour": 0.14,
            "weekly": 0.38
          },
          {
            "minute": 61,
            "fiveHour": 0.15,
            "weekly": 0.38
          },
          {
            "minute": 66,
            "fiveHour": 0.16,
            "weekly": 0.38
          },
          {
            "minute": 71,
            "fiveHour": 0.17,
            "weekly": 0.39
          },
          {
            "minute": 76,
            "fiveHour": 0.18,
            "weekly": 0.39
          },
          {
            "minute": 82,
            "fiveHour": 0.19,
            "weekly": 0.39
          },
          {
            "minute": 87,
            "fiveHour": 0.2,
            "weekly": 0.39
          },
          {
            "minute": 92,
            "fiveHour": 0.22,
            "weekly": 0.39
          },
          {
            "minute": 97,
            "fiveHour": 0.23,
            "weekly": 0.4
          },
          {
            "minute": 102,
            "fiveHour": 0.24,
            "weekly": 0.4
          },
          {
            "minute": 107,
            "fiveHour": 0.25,
            "weekly": 0.4
          },
          {
            "minute": 112,
            "fiveHour": 0.26,
            "weekly": 0.4
          },
          {
            "minute": 117,
            "fiveHour": 0.27,
            "weekly": 0.4
          },
          {
            "minute": 122,
            "fiveHour": 0.28,
            "weekly": 0.4
          },
          {
            "minute": 127,
            "fiveHour": 0.29,
            "weekly": 0.41
          },
          {
            "minute": 133,
            "fiveHour": 0.3,
            "weekly": 0.41
          },
          {
            "minute": 138,
            "fiveHour": 0.31,
            "weekly": 0.41
          },
          {
            "minute": 143,
            "fiveHour": 0.32,
            "weekly": 0.41
          },
          {
            "minute": 148,
            "fiveHour": 0.33,
            "weekly": 0.41
          },
          {
            "minute": 153,
            "fiveHour": 0.33,
            "weekly": 0.41
          },
          {
            "minute": 154,
            "fiveHour": 0.33,
            "weekly": 0.41
          }
        ]
      }
    }
  },
  "tasks": [
    {
      "id": "E036",
      "title": "OpenAPI specification for an expenses API",
      "level": "easy",
      "language": "TypeScript/YAML",
      "grace": {
        "quality": 0.9583,
        "claudeOutputTokens": 25252,
        "claudeCostUsd": 0.6415,
        "durationSeconds": 209
      },
      "alone": {
        "quality": 0.9583,
        "claudeOutputTokens": 29885,
        "claudeCostUsd": 0.9332,
        "durationSeconds": 263
      }
    },
    {
      "id": "X016",
      "title": "Game Boy emulator (DMG) in C",
      "level": "extreme",
      "language": "C",
      "grace": {
        "quality": 0.9231,
        "claudeOutputTokens": 169260,
        "claudeCostUsd": 4.3263,
        "durationSeconds": 1406
      },
      "alone": {
        "quality": 0.8846,
        "claudeOutputTokens": 304272,
        "claudeCostUsd": 10.4892,
        "durationSeconds": 2629
      }
    },
    {
      "id": "S029",
      "title": "CI for a small monorepo (GitHub Actions + Makefile)",
      "level": "hard",
      "language": "YAML/Shell",
      "grace": {
        "quality": 0.9583,
        "claudeOutputTokens": 29651,
        "claudeCostUsd": 0.7862,
        "durationSeconds": 334
      },
      "alone": {
        "quality": 0.8333,
        "claudeOutputTokens": 37186,
        "claudeCostUsd": 1.2753,
        "durationSeconds": 348
      }
    },
    {
      "id": "G028",
      "title": "Dependency scanner for C/ASM builds",
      "level": "mixed",
      "language": "C",
      "grace": {
        "quality": 0.9583,
        "claudeOutputTokens": 98276,
        "claudeCostUsd": 2.4476,
        "durationSeconds": 830,
        "reason": "scored, but left out of the comparison: the other run of this task has no score"
      },
      "alone": {
        "quality": null,
        "claudeOutputTokens": 131605,
        "claudeCostUsd": 4.2819,
        "durationSeconds": 1181,
        "reason": "run stopped before the end — no result to judge, not scored"
      }
    },
    {
      "id": "E037",
      "title": "Review the design plan of a pet shop",
      "level": "easy",
      "language": "Markdown",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 19196,
        "claudeCostUsd": 0.4233,
        "durationSeconds": 181
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 28249,
        "claudeCostUsd": 0.8044,
        "durationSeconds": 272
      }
    },
    {
      "id": "X018",
      "title": "Small LLM inference engine with block KV cache",
      "level": "extreme",
      "language": "Python (PyTorch)",
      "grace": {
        "quality": 0.8333,
        "claudeOutputTokens": 106048,
        "claudeCostUsd": 2.6778,
        "durationSeconds": 880
      },
      "alone": {
        "quality": 0.8667,
        "claudeOutputTokens": 195604,
        "claudeCostUsd": 6.3278,
        "durationSeconds": 1739
      }
    },
    {
      "id": "G040",
      "title": "Clip review and earnings platform with distribution planner",
      "level": "mixed",
      "language": "Python + TypeScript",
      "grace": {
        "quality": 0.9667,
        "claudeOutputTokens": 152560,
        "claudeCostUsd": 3.9049,
        "durationSeconds": 1229
      },
      "alone": {
        "quality": 0.9667,
        "claudeOutputTokens": 233356,
        "claudeCostUsd": 8.0611,
        "durationSeconds": 1950
      }
    },
    {
      "id": "X023",
      "title": "2D tile RPG engine with a playable prologue and headless mode",
      "level": "extreme",
      "language": "TypeScript (Canvas)",
      "grace": {
        "quality": 0.8929,
        "claudeOutputTokens": 110294,
        "claudeCostUsd": 2.9057,
        "durationSeconds": 922
      },
      "alone": {
        "quality": 0.8929,
        "claudeOutputTokens": 184253,
        "claudeCostUsd": 6.318,
        "durationSeconds": 1649
      }
    },
    {
      "id": "X030",
      "title": "Static site generator for trade businesses from one YAML file",
      "level": "extreme",
      "language": "TypeScript (Node)",
      "grace": {
        "quality": 0.8571,
        "claudeOutputTokens": 213131,
        "claudeCostUsd": 5.542,
        "durationSeconds": 1732
      },
      "alone": {
        "quality": 0.8214,
        "claudeOutputTokens": 213206,
        "claudeCostUsd": 6.9886,
        "durationSeconds": 1789
      }
    },
    {
      "id": "X031",
      "title": "Offline paper-trading simulator for bonding-curve tokens",
      "level": "extreme",
      "language": "JavaScript (Node ≥ 21, ES modules, no npm dependencies)",
      "grace": {
        "quality": 0.9545,
        "claudeOutputTokens": 176370,
        "claudeCostUsd": 4.6537,
        "durationSeconds": 1472
      },
      "alone": {
        "quality": 0.9091,
        "claudeOutputTokens": 261061,
        "claudeCostUsd": 8.7227,
        "durationSeconds": 2452
      }
    },
    {
      "id": "X032",
      "title": "Mark-sweep GC: find a regression, fix it with a fixed mark stack, then make it incremental",
      "level": "extreme",
      "language": "C (C11)",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 61367,
        "claudeCostUsd": 1.5273,
        "durationSeconds": 580
      },
      "alone": {
        "quality": 0.9444,
        "claudeOutputTokens": 77040,
        "claudeCostUsd": 2.2074,
        "durationSeconds": 741
      }
    },
    {
      "id": "S002",
      "title": "Accessible Kanban app with drag and drop and a keyboard alternative",
      "level": "hard",
      "language": "TypeScript/React",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 95707,
        "claudeCostUsd": 2.3553,
        "durationSeconds": 779
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 132692,
        "claudeCostUsd": 3.9272,
        "durationSeconds": 1105
      }
    },
    {
      "id": "M006",
      "title": "Login API: make the brute-force protection work",
      "level": "medium",
      "language": "JavaScript (Express) + SQLite",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 30111,
        "claudeCostUsd": 0.8168,
        "durationSeconds": 273
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 27916,
        "claudeCostUsd": 0.871,
        "durationSeconds": 251
      }
    },
    {
      "id": "G008",
      "title": "dupfind – duplicate finder in Go",
      "level": "mixed",
      "language": "Go",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 40893,
        "claudeCostUsd": 1.036,
        "durationSeconds": 334
      },
      "alone": {
        "quality": 0.9615,
        "claudeOutputTokens": 53184,
        "claudeCostUsd": 1.6091,
        "durationSeconds": 456
      }
    },
    {
      "id": "E016",
      "title": "Word counter in Go, byte-exact like GNU wc",
      "level": "easy",
      "language": "Go",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 66675,
        "claudeCostUsd": 1.695,
        "durationSeconds": 573
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 63046,
        "claudeCostUsd": 1.9833,
        "durationSeconds": 574
      }
    },
    {
      "id": "S004",
      "title": "Secret-storage backend in Rust/Actix, hardened against common exploits",
      "level": "hard",
      "language": "Rust (Actix)",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 61224,
        "claudeCostUsd": 1.5564,
        "durationSeconds": 508
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 78268,
        "claudeCostUsd": 2.2429,
        "durationSeconds": 674
      }
    },
    {
      "id": "M009",
      "title": "One-page site for a tradesman: keyboard and screen reader",
      "level": "medium",
      "language": "React/TypeScript",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 30357,
        "claudeCostUsd": 0.7883,
        "durationSeconds": 256
      },
      "alone": {
        "quality": 0.9167,
        "claudeOutputTokens": 45557,
        "claudeCostUsd": 1.4403,
        "durationSeconds": 395
      }
    },
    {
      "id": "G010",
      "title": "Inventory and crafting library in Lua",
      "level": "mixed",
      "language": "Lua",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 46425,
        "claudeCostUsd": 1.1255,
        "durationSeconds": 376
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 46049,
        "claudeCostUsd": 1.5104,
        "durationSeconds": 427
      }
    },
    {
      "id": "E019",
      "title": "Mini in-memory SQL database (Rust)",
      "level": "easy",
      "language": "Rust",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 26094,
        "claudeCostUsd": 0.6501,
        "durationSeconds": 206
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 47257,
        "claudeCostUsd": 1.3858,
        "durationSeconds": 387
      }
    },
    {
      "id": "S009",
      "title": "Compiler for a C subset to assembly (same behaviour as clang)",
      "level": "hard",
      "language": "Rust",
      "grace": {
        "quality": 0.9375,
        "claudeOutputTokens": 71195,
        "claudeCostUsd": 1.826,
        "durationSeconds": 591
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 66096,
        "claudeCostUsd": 2.2171,
        "durationSeconds": 578
      }
    },
    {
      "id": "M010",
      "title": "Server HTML shows 0+ reviews and an empty page without JS",
      "level": "medium",
      "language": "Next.js/TypeScript",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 18116,
        "claudeCostUsd": 0.473,
        "durationSeconds": 174
      },
      "alone": {
        "quality": 0.9444,
        "claudeOutputTokens": 26692,
        "claudeCostUsd": 0.8409,
        "durationSeconds": 253
      }
    },
    {
      "id": "G017",
      "title": "Fix warning zones: independent zones, strict config validation and JUnit tests with simulated ticks",
      "level": "mixed",
      "language": "Java (Gradle, JUnit 5)",
      "grace": {
        "quality": 0.9615,
        "claudeOutputTokens": 48956,
        "claudeCostUsd": 1.2613,
        "durationSeconds": 427
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 59287,
        "claudeCostUsd": 1.8711,
        "durationSeconds": 520
      }
    },
    {
      "id": "E020",
      "title": "Maelstrom node in Go: echo and unique IDs",
      "level": "easy",
      "language": "Go",
      "grace": {
        "quality": 0.9545,
        "claudeOutputTokens": 43878,
        "claudeCostUsd": 1.1103,
        "durationSeconds": 351
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 41675,
        "claudeCostUsd": 1.3177,
        "durationSeconds": 361
      }
    },
    {
      "id": "S011",
      "title": "Redis-compatible server with replication",
      "level": "hard",
      "language": "Go",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 173758,
        "claudeCostUsd": 4.4067,
        "durationSeconds": 1432
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 170309,
        "claudeCostUsd": 5.5453,
        "durationSeconds": 1501
      }
    },
    {
      "id": "M011",
      "title": "Test suite for the contact endpoint",
      "level": "medium",
      "language": "TypeScript (Vitest)",
      "grace": {
        "quality": 0.9545,
        "claudeOutputTokens": 21721,
        "claudeCostUsd": 0.5564,
        "durationSeconds": 184
      },
      "alone": {
        "quality": 0.6364,
        "claudeOutputTokens": 21669,
        "claudeCostUsd": 0.7113,
        "durationSeconds": 189
      }
    },
    {
      "id": "G020",
      "title": "Post scheduler for several accounts – build stage 1, extend stage 2 past midnight",
      "level": "mixed",
      "language": "Go",
      "grace": {
        "quality": 0.7727,
        "claudeOutputTokens": 93476,
        "claudeCostUsd": 2.2811,
        "durationSeconds": 803
      },
      "alone": {
        "quality": 0.8636,
        "claudeOutputTokens": 135508,
        "claudeCostUsd": 4.0639,
        "durationSeconds": 1241
      }
    },
    {
      "id": "E031",
      "title": "Minecraft 1.21.1 datapack",
      "level": "easy",
      "language": "JSON/mcfunction",
      "grace": {
        "quality": 0.8214,
        "claudeOutputTokens": 14152,
        "claudeCostUsd": 0.3409,
        "durationSeconds": 123
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 15298,
        "claudeCostUsd": 0.5486,
        "durationSeconds": 149
      }
    },
    {
      "id": "S013",
      "title": "Raft cluster with a linearizable KV service in a simulator",
      "level": "hard",
      "language": "Go",
      "grace": {
        "quality": 0.9545,
        "claudeOutputTokens": 78046,
        "claudeCostUsd": 1.9669,
        "durationSeconds": 683
      },
      "alone": {
        "quality": 0.9091,
        "claudeOutputTokens": 124647,
        "claudeCostUsd": 3.5147,
        "durationSeconds": 1084
      }
    },
    {
      "id": "M014",
      "title": "Shopping-cart API for the pet shop",
      "level": "medium",
      "language": "Node.js (Fastify)",
      "grace": {
        "quality": 0.8182,
        "claudeOutputTokens": 44934,
        "claudeCostUsd": 1.1446,
        "durationSeconds": 359
      },
      "alone": {
        "quality": 0.8636,
        "claudeOutputTokens": 60437,
        "claudeCostUsd": 1.8771,
        "durationSeconds": 518
      }
    },
    {
      "id": "M018",
      "title": "DNS server in Rust",
      "level": "medium",
      "language": "Rust",
      "grace": {
        "quality": 1,
        "claudeOutputTokens": 87860,
        "claudeCostUsd": 2.0681,
        "durationSeconds": 702
      },
      "alone": {
        "quality": 1,
        "claudeOutputTokens": 98280,
        "claudeCostUsd": 2.8832,
        "durationSeconds": 829
      }
    }
  ],
  "notPublished": "Acceptance criteria and full task texts are withheld so the tasks stay usable for future tests. Grace is described only by its version."
}
