From 4b2f9d54d88f5a0e7ecd407a8b2ff17bbe35878c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 11 Nov 2026 11:39:16 -0700 Subject: [PATCH 1/4] test(oracle-gaps): bound the e2e tail; one edit sequence over the witness module A generated history with the witness module takes no steps: each step adds a cold run of the witness module, and about one campaign in six drew a history past the test's limit. The constructed history "an edit sequence over the witness module" (test, helper, revert) keeps law 9 judged over the witness module's outcome classes in every campaign, and enters the new required state "a timed-out run reused after an edit". The grammar narrows through a map with an unmapper, so fast-check still takes every history back. The limit is stated in the header and in law.limits (maintainer decision, 2026-11-11, option M). Held until D: the construction costs about 160 s locally, and its first fixed-seed run found an intermittent law 9 violation (a stale daemon kill after the helper edit), which is fixed separately. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 47 +++++++++- .../gap-report-oracle.property.test.mjs | 90 ++++++++++++++++++- 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 0031a37..86d5522 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -434,7 +434,7 @@ const daemonEvents = new Set([ `resetBeforeRead`, `otherInstall`, ]) -const histories = fc.record({ +const drawn = fc.record({ functions: fc.array(fc.array(statement, { minLength: 1, maxLength: 5 }), { minLength: 1, maxLength: 3, @@ -550,7 +550,6 @@ const histories = fc.record({ `insertGuard`, `reformat`, ), - // Most histories have no steps, which keeps the campaign's cost down. // Whether the run asks for the ordinary tests' outcome (`--ordinary`). ordinary: fc.boolean(), // A second authored mutant, on the next statement, so that two patched @@ -633,6 +632,8 @@ const histories = fc.record({ { weight: 29, arbitrary: fc.constant(4) }, { weight: 1, arbitrary: fc.constant(11) }, ), + // Most histories have no steps, which keeps the campaign's cost down. A + // history with the witness module has none (see `histories`). steps: fc.oneof( { weight: 5, arbitrary: fc.constant(null) }, { @@ -658,6 +659,47 @@ const histories = fc.record({ importer: fc.constantFrom(`none`, `a`, `b`), }) +// The edit sequence over the witness module, the one list of steps that a +// history with the witness module takes. A test edit makes the daemon +// recompute file a's runs, the hangs among them; the helper edit changes +// only file b, so the daemon reuses file a's timed-out runs; the revert +// returns to a state that the daemon has seen. +const witnessSteps = [`test`, `helper`, `revert`] +const isWitnessSteps = (steps) => + steps !== null && + steps.length === witnessSteps.length && + steps.every((step, i) => step === witnessSteps[i]) + +// The histories. The witness module makes each cold run about 20 s (its +// hangs, slow kills, and stability rerun), and each step adds a cold run and +// a daemon check, so a generated history with both could run past the +// test's time limit: about one campaign in six drew a witness history with +// eight or more steps. A generated history with the witness module +// therefore has no steps. The constructed history "an edit sequence over the +// witness module" takes `witnessSteps`, so every campaign still judges law +// 9 over the witness module's outcome classes, along one short sequence +// (maintainer decision, 2026-11-11, option M of the CI plan; a stated +// limit of the property). The map is its own inverse on the histories it +// keeps, so fast-check takes every one of them back and shrinks it. +const histories = drawn.map( + (history) => + history.witness === false || + history.steps === null || + isWitnessSteps(history.steps) + ? history + : { ...history, steps: null }, + (value) => { + const history = /** @type {{ witness: unknown, steps: unknown }} */ (value) + if ( + history.witness !== false && + history.steps !== null && + !isWitnessSteps(history.steps) + ) + throw new Error(`a history with the witness module takes no steps`) + return value + }, +) + // The witness module: functions whose outcome classes generated histories // reach rarely, each with its own owners and tests. const kill = { kind: `if`, op: `>`, k: 4, not: false, quote: `'` } @@ -4587,6 +4629,7 @@ export { edits, daemonEvents, histories, + witnessSteps, kill, push, judging, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index cf856b4..530efcd 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -298,6 +298,13 @@ * reach agreement kills every probe mutant is not checked. Gap answers and * the gate are not part of the CLI until answers become evidence-log events, * which waits on the maintainer's decision about how an answer maps to a law. + * Law 9 judges the witness module's outcome classes along one edit + * sequence only, the constructed history "an edit sequence over the witness + * module" (a test edit, a helper edit, and a revert): a generated history + * with the witness module takes no steps, so longer or other edit sequences + * over those classes are not drawn. Each such step costs a cold run of the + * witness module, and one campaign in six drew a history that ran past the + * test's limit (maintainer decision, 2026-11-11; this oracle owns the gap). * Law 9's edits do not include a file that a test reads through `fs`, an * environment variable other than the seed, a new file that changes how an * unchanged import resolves, a change to the Vite config or the installed @@ -380,6 +387,7 @@ import { edits, daemonEvents, histories, + witnessSteps, push, judging, witness, @@ -914,6 +922,18 @@ async function incremental(dir, plan, cli, copy, observe, states) { states.add(`incremental check reused some evidence and recomputed some`) if (step === `restart` && reused > 0) states.add(`evidence reused after a daemon restart`) + // The witness module's hangs are cut off in every cold run. A daemon + // check after an edit that cut off fewer runs than the cold run of the + // same state reused a timed-out run, so law 9 judged a timeout that came + // from the store. + if ( + plan.witness !== false && + step !== `start` && + !daemonEvents.has(step) && + reused > 0 && + (daemon.evidence?.cut ?? 0) < (coldRun.evidence?.cut ?? 0) + ) + states.add(`a timed-out run reused after an edit`) // Undoing an edit to a file returns to a state the daemon has seen, so // it reruns nothing. if ( @@ -2002,6 +2022,7 @@ const required = [ `a session failure larger than a socket buffer`, `a config fault in a run that owns nothing`, ...incrementalStates, + `a timed-out run reused after an edit`, ] // The faults the CLI analyzes anyway: a package's TypeScript is no input. const analyzedFaults = [`noTypescript`, `apiLessTypescript`] @@ -2053,6 +2074,7 @@ const separate = [ `a run with eleven contexts`, `a failing owner replayed under a config that --config names`, ...incrementalStates, + `a timed-out run reused after an edit`, ] const witnessed = required.filter((s) => !separate.includes(s)) @@ -2079,7 +2101,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs.`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 6 s full limit, and a slow run only as one witness that waits 3 s before it judges; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; no --only, --sample, or --diff-base; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only four or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; a workspace dependency's one-file bundle is written in that dependency's own .oracle-gaps folder, a known stray write (date-fns report), which the stray-write check excludes; the runner's copies and Vitest configs (.oracle-gaps-PID*) sit at the package root and are removed when the run closes, so a killed run leaves them there, and the stray-write check runs after close and cannot see them; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 6 s full limit, and a slow run only as one witness that waits 3 s before it judges; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; no --only, --sample, or --diff-base; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only four or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), because a generated history with the witness module takes no steps (maintainer decision, 2026-11-11; this oracle owns the gap); law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; a workspace dependency's one-file bundle is written in that dependency's own .oracle-gaps folder, a known stray write (date-fns report), which the stray-write check excludes; the runner's copies and Vitest configs (.oracle-gaps-PID*) sit at the package root and are removed when the run closes, so a killed run leaves them there, and the stray-write check runs after close and cannot see them; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -2481,10 +2503,74 @@ oracleProperty({ ...generated, fault, ...(fault === `brokenVitest` - ? { steps: [`restart`, `fixSession`], concurrentStart: true } + ? { + steps: [`restart`, `fixSession`], + concurrentStart: true, + // A history with the witness module takes no other steps. + witness: false, + } : {}), }), })), + { + // Law 9 over the witness module's outcome classes: its hangs are + // recomputed after the test edit and reused after the helper edit, + // which changes only file b. Generated histories with the witness + // module take no steps, so this is the one sequence that judges + // them (maintainer decision, 2026-11-11). Everything else is the + // cheapest legal setting, so that the history costs about two cold + // runs and four daemon checks of the witness module. + name: `an edit sequence over the witness module`, + enters: [`a timed-out run reused after an edit`], + build: (generated) => ({ + ...generated, + functions: [ + [{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }], + [{ kind: `push`, quote: `'` }], + ], + containers: [], + overloads: [], + owners: [`a`, `b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 5), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 1 }, + ], + witness: `capture`, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + config: `object`, + configEnv: false, + setupGlobal: false, + native: `none`, + depSubpath: false, + unbuiltDep: false, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + configFile: `vite.config.ts`, + configPlace: `discovered`, + packageType: `module`, + configAwait: false, + configRequire: false, + contexts: 4, + sourceSpelling: `plain`, + steps: witnessSteps, + }), + }, ]), }, resolve: resolveHistory, From f5194546d9fa26feb7f124f4f4a13221c229164a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 11 Nov 2026 20:35:28 -0700 Subject: [PATCH 2/4] ci(oracles): put the witness edit sequence's units in the e2e shards The constructed history "an edit sequence over the witness module" is a fixed-seed, a random, and a determinism unit. They go to e2e-4, e2e-5, and e2e-1, measured at 97 s, about 97 s, and 189 s locally under load. That puts those shards near 620 to 710 s in the measured run's terms, past the packing's 522 s, so the shards need a new longest-first packing from a full run before CI runs them. Co-authored-by: Isaac --- packages/oracles/scripts/e2e-shards.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 1209d49..9db41db 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -59,6 +59,9 @@ export const shards = { `determinism: constructed history: a declaration fault: projects`, `determinism: constructed history: a package with no config`, `determinism: constructed history: an edit sequence through the daemon`, + // The witness module's edit sequence (option M): 189 s here, measured + // with the daemon fix of fix/daemon-current-inputs. + `determinism: constructed history: an edit sequence over the witness module`, ], // 522 s in the measured run, about 221 s idle. 'e2e-2': [ @@ -125,6 +128,9 @@ export const shards = { `determinism: constructed history: an ambiguous authored mutant`, `determinism: constructed history: every outcome class`, `determinism: constructed history: two authored mutants`, + // The witness module's edit sequence (option M): 97 s here, measured + // with the daemon fix of fix/daemon-current-inputs. + `fixed seed 20261105: an edit sequence over the witness module`, ], // 522 s in the measured run, about 221 s idle. 'e2e-5': [ @@ -148,6 +154,9 @@ export const shards = { `determinism: constructed history: a declaration fault: noTypescript`, `determinism: constructed history: a failing owner under a config that --config names`, `determinism: generated 2`, + // The witness module's edit sequence (option M): about 97 s here, measured + // with the daemon fix of fix/daemon-current-inputs. + `random seed: an edit sequence over the witness module`, ], } From 2d7e13c2a35393cd5e6fcc3cb86e1e639a5a9d57 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 11 Nov 2026 20:36:13 -0700 Subject: [PATCH 3/4] ci(oracles): the shard notes say that option M bounds the random tail Co-authored-by: Isaac --- packages/oracles/scripts/e2e-shards.mjs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 9db41db..d701afa 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -33,10 +33,11 @@ const suite = `oracle-gaps.report` // (units), so a shard can hold any of them; the slowest unit, the // determinism of the edit sequence, is 236 s here. The shards are a // longest-first packing of those costs, listed by kind. A random unit's -// cost follows its seed, so a shard's time varies from run to run. CI keeps -// its one fixed-seed job until the random units' tail is bounded (option M -// of the e2e speed plan): a random unit that draws a long edit sequence over -// the witness module can pass 10 minutes in CI. +// cost follows its seed, so a shard's time varies from run to run. Option M +// of the e2e speed plan bounds the random units' tail: a generated history +// with the witness module takes no steps, and the one edit sequence over it +// is a constructed history. CI keeps its one fixed-seed job until the +// maintainer decides which units run on a pull request. export const shards = { // 522 s in the measured run, about 221 s idle. 'e2e-1': [ From 2cc83166d4e2d065717daae2870173018bbe98ce Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 00:13:39 -0700 Subject: [PATCH 4/4] ci(oracles): the first fixed-seed shard takes the two edit sequences With the witness module's edit sequence (85 s locally) and the generated witness histories now taking no steps, the fixed-seed units cost 245 s; the two edit sequences are 130 s of it, the other 30 units 115 s. Co-authored-by: Isaac --- packages/oracles/scripts/pr-shards.mjs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index a2acbde..5421f93 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -52,10 +52,11 @@ export const partitions = { }, }, // The end-to-end gap oracle's fixed-seed units, which a pull request runs - // (`-t "fixed seed"`): 31 units, 325 s in all. A unit's cost follows the - // generated history its constructed history is built from, so a grammar - // change can move cost between units. The first shard takes the units - // that cost most at 908b1f9: 168 s and 157 s. + // (`-t "fixed seed"`): 32 units, 245 s in all, measured on the witness + // edit-sequence branch (option M) at load average 4. A unit's cost follows + // the generated history its constructed history is built from, so a + // grammar change can move cost between units. The first shard takes the + // two edit sequences, 85 s and 45 s; the second the rest, 115 s. 'oracle-gaps-fixed-seed': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], suite: `oracle-gaps.report`, @@ -65,11 +66,8 @@ export const partitions = { previously: `fixed seed`, shards: { 1: anyOf([ - `a declaration fault: missingSource`, + `an edit sequence over the witness module`, `an edit sequence through the daemon`, - `a declaration fault: configNoDefault`, - `an edit that changed the cold result after a hang replaced the only context`, - `a lazy module loaded before a hang in the same batch`, ]), 2: rest, },