Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelog/unreleased/1749.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
- **BREAKING CHANGES:** `emergent_testing`: `Reporter.result` now receives the
normalized `TestResult` and `Reporter.summary` one `RunTotals` record; the
new `addResult` and `zeroTotals` fold leaf results into every runner's
totals. Printed output and the browser report are unchanged
39 changes: 26 additions & 13 deletions fjs/emergent_testing/browser.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
* @import { Result } from '../types/result/types.ts'
*/

import { collectTests, testResult } from './module.f.mjs'
import { addResult, collectTests, testResult, zeroTotals } from './module.f.mjs'
import { error as errorResult, invert, ok } from '../types/result/module.f.mjs'

/** @type {(value: unknown) => string} */
Expand Down Expand Up @@ -200,13 +200,24 @@ const runOne = (module, path, throws, fn, result) => {
return Promise.resolve().then(() => [fn()]).then(([value]) => settled(value), failed)
}

/** @type {(status: string, duration: number, results: readonly _BrowserTestResult[]) => BrowserTestReport} */
const reportOf = (status, duration, results) => {
const failed = results.filter(result => result.status === 'failed').length
/**
* The run-ended event, as the page reports it. The counts — and with them the
* run's own pass/fail status — come from folding the results with the same
* `addResult` that decides `fjs t`'s summary and exit code, so "did the run
* pass" has one answer across the runners. `duration` stays the page's own
* wall clock: leaves run concurrently here, so the fold's summed duration is
* not how long the run took (see `RunTotals`).
*
* `status` overrides the folded decision when the run never got to its leaves
* — module loading failed — which no leaf result can express.
*
* @type {(duration: number, results: readonly _BrowserTestResult[], status?: string) => BrowserTestReport} */
const reportOf = (duration, results, status = undefined) => {
const { passed, failed } = results.reduce(addResult, zeroTotals)
return {
status,
status: status ?? (failed !== 0 ? 'failed' : 'passed'),
browser: navigator.userAgent,
totals: { tests: results.length, passed: results.length - failed, failed },
totals: { tests: results.length, passed, failed },
duration,
results,
}
Expand All @@ -215,6 +226,11 @@ const reportOf = (status, duration, results) => {
/**
* Runs named proof exports and returns the serializable browser report.
*
* `result` is the page's subscription to the leaf-landed event — the same
* event `fjs t`'s `Reporter.result` carries, a shared `TestResult` plus the
* browser's own `message`/`stack` part — and the resolved report is its
* run-ended event, with totals folded by the shared `addResult`.
*
* @type {(modules: readonly (readonly [string, unknown])[], result?: (result: _BrowserTestResult) => void) => Promise<BrowserTestReport>}
*/
export const runBrowserProofs = (modules, result = () => undefined) => {
Expand Down Expand Up @@ -259,11 +275,7 @@ export const runBrowserProofs = (modules, result = () => undefined) => {
).then(next => runBatch(index + batchSize, next))
}
const completed = runBatch(0, [])
return completed.then(results => reportOf(
results.some(result => result.status === 'failed') ? 'failed' : 'passed',
performance.now() - start,
results,
))
return completed.then(results => reportOf(performance.now() - start, results))
}

/** @typedef {(source: string) => Promise<{ readonly proof?: unknown }>} _BrowserImporter */
Expand Down Expand Up @@ -337,11 +349,12 @@ export const startBrowserTestSources = (root, sources, importer) => {
// that disagreed with `results` would tell an automated consumer
// the suite was empty rather than broken.
const duration = performance.now() - start
return publish(root, Promise.resolve(reportOf('infrastructure-error', duration,
return publish(root, Promise.resolve(reportOf(duration,
rejected.map(({ source, error }) => {
const [message, stack] = errorDetails(error)
return moduleFailure(source, duration, message, stack)
}))))
}),
'infrastructure-error')))
}
return startBrowserTests(root, loadedModules.flatMap(module =>
module.status === 'loaded'
Expand Down
100 changes: 57 additions & 43 deletions fjs/emergent_testing/module.f.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
*
* Two parallel execution paths:
* - `runModule` / `Reporter<O>` — self-hosted Effects runner used by `fjs t`;
* sandboxes each leaf call individually and accumulates `TestState`.
* sandboxes each leaf call individually and accumulates `RunTotals`.
* - `registerModule` / `TestContext` — registers tests with an external
* framework (Node `--test`, Bun, Deno) at import time; the framework owns
* scheduling and pass/fail counting.
Expand All @@ -13,7 +13,7 @@
* @import { Operation } from '../effects/types.ts'
* @import { Effect, NotImplemented } from '../effects/types.ts'
* @import { LoadModuleOperations, ModuleMap } from '../dev/types.ts'
* @import { TestFn, TestEntry, TestSet, Path, Reporter, TestResult, _TestState, _TestAndPath } from './types.ts'
* @import { TestFn, TestEntry, TestSet, Path, Reporter, RunTotals, TestResult, _TestAndPath } from './types.ts'
* @import { All, Await, Env, IoChannel, NodeProgram, NodeProgramOptions, Program, Sandbox, SandboxResult, Test, TestContext, Write, WriteConsoles } from '../effects/node/types.ts'
*/

Expand All @@ -26,13 +26,28 @@ import { loadModuleMap } from '../dev/module.f.mjs'
import { invert } from '../types/result/module.f.mjs'
import { definedEntries } from '../types/object/module.f.mjs'

/** @type {(delta: number) => (ts: _TestState) => _TestState} */
const addPass = delta => ts =>
({ ...ts, time: ts.time + delta, pass: ts.pass + 1 })
/**
* The empty {@link RunTotals}: what a run's totals are before any leaf lands.
*
* @type {RunTotals}
*/
export const zeroTotals = { passed: 0, failed: 0, duration: 0 }

/** @type {(delta: number) => (ts: _TestState) => _TestState} */
const addFail = delta => ts =>
({ ...ts, time: ts.time + delta, fail: ts.fail + 1 })
/**
* Folds one leaf-landed event into a run's totals.
*
* This is where "did the run pass" is decided, for every runner: the counts
* come from each result's shared `status`, so the summary line, the exit code
* and the browser report's totals all read the same fold of the same events
* rather than each counting their own way.
*
* @type {(totals: RunTotals, r: TestResult) => RunTotals}
*/
export const addResult = (totals, r) => ({
passed: totals.passed + (r.status === 'passed' ? 1 : 0),
failed: totals.failed + (r.status === 'failed' ? 1 : 0),
duration: totals.duration + r.duration,
})
Comment thread
sergey-shandar marked this conversation as resolved.

/** @type {(a: number) => string} */
const timeFormat = a => {
Expand Down Expand Up @@ -154,49 +169,49 @@ export const registerModule = (ctx, k, v, star) => {
return mapStep(allOk(...tests.map(e => registerOne(ctx, e))), () => undefined)
}

/** @type {(a: _TestState, b: _TestState) => _TestState} */
const mergeState = (a, b) =>
({ time: a.time + b.time, pass: a.pass + b.pass, fail: a.fail + b.fail })

/** @type {_TestState} */
const zero = { time: 0, pass: 0, fail: 0 }
/** @type {(a: RunTotals, b: RunTotals) => RunTotals} */
const mergeTotals = (a, b) =>
({ passed: a.passed + b.passed, failed: a.failed + b.failed, duration: a.duration + b.duration })

/**
* @template {Operation} O
* @param {Reporter<O>} reporter
* @returns {(k: string, v: unknown) => (ts: _TestState) => Effect<O | All, _TestState, IoChannel>}
* @returns {(k: string, v: unknown) => (ts: RunTotals) => Effect<O | All, RunTotals, IoChannel>}
*/
const runModule = ({ result, test }) => (k, v) => ts => {
/** @type {(entry: _TestAndPath) => Effect<O | All, _TestState, IoChannel>} */
/** @type {(entry: _TestAndPath) => Effect<O | All, RunTotals, IoChannel>} */
const one = ([testPath, set]) => {
// The sandbox result is still needed after it has been reported, so the
// reporting call is captured rather than nested inside its own step.
// The leaf's shared record is built here, next to the sandbox result it
// is read from, so the leaf-landed event carries the value already
// decided — a reporter renders `t`, it does not derive its own.
const evaluated = mapStep(
test(k, testPath, set),
sr => /** @type {const} */ ([testResult(k, testPath, sr), sr]))
// Both are still needed after they have been reported, so the reporting
// call is captured rather than nested inside its own step.
const reported = historyStep(
history(test(k, testPath, set)),
sr => result(k, testPath, sr, set.throws))
history(evaluated),
([t, sr]) => result(t, sr, set.throws))
return step(
reported,
([, sr]) => {
const { result: [s, r], duration } = sr
if (s !== 'ok') {
return pureOk(addFail(duration)(zero))
}
if (set.throws) {
return pureOk(addPass(duration)(zero))
([, [t, sr]]) => {
const total = addResult(zeroTotals, t)
if (t.status !== 'passed' || set.throws) {
return pureOk(total)
}
// Walk return-value sub-tree; null marks the call boundary so
// paths render as e.g. `outer().inner`. throws resets to false.
return mapStep(
walk([...testPath, null], false, r),
sub => mergeState(addPass(duration)(zero), sub))
walk([...testPath, null], false, sr.result[1]),
sub => mergeTotals(total, sub))
})
}
/** @type {(path: Path, throws: boolean, v: unknown) => Effect<O | All, _TestState, IoChannel>} */
/** @type {(path: Path, throws: boolean, v: unknown) => Effect<O | All, RunTotals, IoChannel>} */
const walk = (path, throws, v) => {
const effects = collectTests(path, throws, v).map(one)
return mapStep(allOk(...effects), states => states.reduce(mergeState, zero))
return mapStep(allOk(...effects), states => states.reduce(mergeTotals, zeroTotals))
}
return mapStep(walk([], false, v), delta => mergeState(ts, delta))
return mapStep(walk([], false, v), delta => mergeTotals(ts, delta))
}

/** @type {(moduleMap: ModuleMap) => readonly (readonly [string, unknown])[]} */
Expand All @@ -217,15 +232,15 @@ export const runModuleMap = reporter => moduleMap => {
const { summary } = reporter
const modules = proofEntries(moduleMap)
const total = mapStep(
allOk(...modules.map(([k, v]) => runModule(reporter)(k, v)(zero))),
m => m.reduce(mergeState, zero))
allOk(...modules.map(([k, v]) => runModule(reporter)(k, v)(zeroTotals))),
m => m.reduce(mergeTotals, zeroTotals))
// The totals are still needed after the summary has been printed, so they
// are carried forward in a history rather than closed over by a nested
// continuation.
const reported = historyStep(
history(total),
ts => summary(ts.pass, ts.fail, ts.time))
return mapStep(reported, ([, ts]) => ts.fail !== 0 ? 1 : 0)
summary)
return mapStep(reported, ([, ts]) => ts.failed !== 0 ? 1 : 0)
}

/**
Expand Down Expand Up @@ -423,25 +438,24 @@ export const defaultReporter = options => {
const isGitHub = options.env['GITHUB_ACTIONS'] !== undefined
return {
// https://github.com/OndraM/ci-detector/blob/main/src/Ci/GitHubActions.php
result: (file, path, r, throws) => {
const t = testResult(file, path, r)
result: (t, r, throws) => {
const v = r.result[1]
return t.status === 'passed'
? csiLog(fmtResultLine(t, fgGreen, 'ok') + (throws ? ' # EXPECTED TO THROW' : ''))
: isGitHub
? csiError(`::error file=${file},line=1,title=${ghEscape(t.name)}::${ghEscape(String(v))}`)
? csiError(`::error file=${t.module},line=1,title=${ghEscape(t.name)}::${ghEscape(String(v))}`)
// `step`, so the detail line is attempted only when the
// header line was written: two halves of one report, and
// half of it is worse than none.
: step(
csiError(fmtResultLine(t, fgRed, 'error')),
() => csiError(`${fgRed}${v}${reset}`))
},
summary: (pass, fail, time) => {
const fgFail = fail === 0 ? fgGreen : fgRed
summary: ({ passed, failed, duration }) => {
const fgFail = failed === 0 ? fgGreen : fgRed
return step(
csiLog(`${bold}Number of tests: pass: ${fgGreen}${pass}${reset}${bold}, fail: ${fgFail}${fail}${reset}${bold}, total: ${pass + fail}${reset}`),
() => csiLog(`${bold}Time: ${timeFormat(time)}${reset}`))
csiLog(`${bold}Number of tests: pass: ${fgGreen}${passed}${reset}${bold}, fail: ${fgFail}${failed}${reset}${bold}, total: ${passed + failed}${reset}`),
() => csiLog(`${bold}Time: ${timeFormat(duration)}${reset}`))
},
test: defaultTest,
}
Expand Down
Loading
Loading