From e39c7825ba0d7c3c6ae3444d20de54b4dbbfc45e Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 12 Jul 2026 22:22:47 +0200 Subject: [PATCH 01/81] Function for calculating hashes --- lib/Hash.js | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 lib/Hash.js diff --git a/lib/Hash.js b/lib/Hash.js new file mode 100644 index 00000000..8b7d0c32 --- /dev/null +++ b/lib/Hash.js @@ -0,0 +1,117 @@ +const crypto = require('crypto'); + +/** + * The compiled Elm JavaScript is basically just a long sequence of definitions. + * Some are `function` statements, some are `var` assignments. + * + * Values that use themselves inside themselves in certain ways are defined + * with `function $some$module$cyclic$functionName`, and wrapped in `try {}` + * during development – that’s the only time a definition can be indented. + * + * We also need to support a `}` at the start of the line, because I’ve seen this + * code being generated (in https://github.com/lydell/codebase-ui/tree/02eac5056da3283687e0b61fa94a30ca6f71e3fb): + * + * function _Http_track(router, xhr, tracker) + * { + * // stuff + * }var $author$project$PreApp$AppMsg = function (a) { + * return {$: 'AppMsg', a: a}; + * }; + */ +const CHUNK_REGEX = /^(?=\}?(?:var|function|try))/m; + +/** + * Companion to `CHUNK_REGEX`. Extracts the name of the thing being defined a chunk. + * Remember that the chunk may start with `try` and be indented. + */ +const CHUNK_DEFINITION_NAME_REGEX = /(?:var|function) ([^ (]+)/; + +/** + * Matches string literals, multiline comments, singleline comments and some identifiers - + * which may be references to other chunks. Such references must start with either a + * dollar sign or an underscore. We only care about out the identifiers, but match the + * other literals too, so that we don’t get false positives for identifiers inside strings and comments. + * Parts copied from: https://github.com/lydell/js-tokens/blob/895fb4d6804a287aecfb0e1009851f925d07b079/index.coffee + */ +const REFERENCES_REGEX = + /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$_\u200C\u200D\p{ID_Continue}]+/gu; + +/** + * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) + * which may be tests. For each fully qualified name, find the corresponding + * JavaScript definition in the compiled Elm JavaScript code, and return a + * hash of the code of that definition. If the definition refers to other + * definition (it calls other functions), the hash is based on both the hash + * of the code of the definition, and of the hashes of all referenced definitions. + * + * This way we can tell if the code that will be running via an exposed `Test` + * value has changed or not, and thus if we need to re-run it or not. + * + * @param { Array> } fullyQualifiedNames + * @param { string } code + * @returns { Array<{ name: Array, hash: string }> } + */ +function calculateHashes(fullyQualifiedNames, code) { + /** @type { Record } */ + const chunks = Object.fromEntries( + code.split(CHUNK_REGEX).flatMap((chunk) => { + const match = CHUNK_DEFINITION_NAME_REGEX.exec(chunk); + // Not all chunks contain a definition. + if (match === null) { + return []; + } + const name = match[1]; + return [[name, chunk]]; + }) + ); + + /** @type { Record } */ + const hashes = {}; + + /** + * @param { string } name + * @param { Array } seen + * @returns { string } + */ + const getOrCalculateHash = (name, seen) => { + const hash = hashes[name]; + if (hash !== undefined) { + return hash; + } + const chunk = chunks[name]; + if (chunk === undefined) { + const newHash = ''; + hashes[name] = newHash; + return newHash; + } + const references = chunk.match(REFERENCES_REGEX); + if (references === null) { + throw new Error(`No references found in chunk for ${name}:\n${chunk}`); + } + // When testing on a large project, all hashes led to about the same + // amount of time used by `getOrCalculateHash`. `sha256` is one of + // the ones being about 10 ms faster than the slowest ones. + const hashObject = crypto.createHash('sha256'); + hashObject.update(chunk); + for (const reference of references) { + if ( + (reference.startsWith('$') || reference.startsWith('_')) && + !seen.includes(reference) + ) { + hashObject.update(getOrCalculateHash(reference, [reference, ...seen])); + } + } + const newHash = hashObject.digest('hex'); + hashes[name] = newHash; + return newHash; + }; + + return fullyQualifiedNames.map((fullyQualifiedName) => { + const name = `$author$project$${fullyQualifiedName.join('$')}`; + return { name: fullyQualifiedName, hash: getOrCalculateHash(name, [name]) }; + }); +} + +module.exports = { + calculateHashes, +}; From ab3e620c03994062f71051634720fd7caa4e0741 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Mon, 13 Jul 2026 22:34:00 +0200 Subject: [PATCH 02/81] Handle recursion correctly --- lib/Hash.js | 107 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 86 insertions(+), 21 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index 8b7d0c32..e1e994ae 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -36,6 +36,14 @@ const CHUNK_DEFINITION_NAME_REGEX = /(?:var|function) ([^ (]+)/; const REFERENCES_REGEX = /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$_\u200C\u200D\p{ID_Continue}]+/gu; +/** + * @param { Array } fullyQualifiedName + * @returns { string } + */ +function fullyQualifiedNameToVariableName(fullyQualifiedName) { + return `$author$project$${fullyQualifiedName.join('$')}`; +} + /** * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) * which may be tests. For each fully qualified name, find the corresponding @@ -65,41 +73,98 @@ function calculateHashes(fullyQualifiedNames, code) { }) ); + /** + * @typedef { { code: Array, references: Set } } Item + * @type { Record } + */ + const items = {}; + + /** + * @param { string } name + * @param { string } chunk + * @param { Array } seen + * @returns { void } + */ + const update = (name, chunk, seen) => { + if (name in items) { + return; + } + const references = chunk.match(REFERENCES_REGEX); + if (references === null) { + throw new Error(`No references found in chunk for ${name}:\n${chunk}`); + } + /** @type { Item } */ + const item = { + code: [chunk], + references: new Set( + references.filter( + (reference) => + (reference.startsWith('$') || reference.startsWith('_')) && + reference !== name && + reference in chunks + ) + ), + }; + items[name] = item; + const newSeen = [...seen, name]; + for (const reference of item.references) { + const referenceChunk = chunks[reference]; + // TODO: Check if undefined or not? + let index = newSeen.indexOf(reference); + if (index === -1) { + update(reference, referenceChunk, newSeen); + } else { + const chain = newSeen.slice(index); + /** @type { Item } */ + const newItem = { code: [], references: new Set() }; + for (const name2 of chain) { + const item2 = items[name2]; + newItem.code.push(...item2.code); + for (const n of item2.references) { + if (!chain.includes(n)) { + newItem.references.add(n); + } + } + items[name2] = newItem; + } + } + } + }; + + for (const fullyQualifiedName of fullyQualifiedNames) { + const name = fullyQualifiedNameToVariableName(fullyQualifiedName); + const chunk = chunks[name]; + if (chunk === undefined) { + throw new Error(`Could not find ${name} in the compiled code!`); + } + update(name, chunk, []); + } + /** @type { Record } */ const hashes = {}; /** * @param { string } name - * @param { Array } seen * @returns { string } */ - const getOrCalculateHash = (name, seen) => { + const getOrCalculateHash = (name) => { const hash = hashes[name]; if (hash !== undefined) { return hash; } - const chunk = chunks[name]; - if (chunk === undefined) { - const newHash = ''; - hashes[name] = newHash; - return newHash; - } - const references = chunk.match(REFERENCES_REGEX); - if (references === null) { - throw new Error(`No references found in chunk for ${name}:\n${chunk}`); + const item = items[name]; + if (item === undefined) { + throw new Error(`No item for ${name}!`); } // When testing on a large project, all hashes led to about the same // amount of time used by `getOrCalculateHash`. `sha256` is one of // the ones being about 10 ms faster than the slowest ones. const hashObject = crypto.createHash('sha256'); - hashObject.update(chunk); - for (const reference of references) { - if ( - (reference.startsWith('$') || reference.startsWith('_')) && - !seen.includes(reference) - ) { - hashObject.update(getOrCalculateHash(reference, [reference, ...seen])); - } + for (const code of Array.from(item.code).sort()) { + hashObject.update(code); + } + for (const reference of Array.from(item.references).sort()) { + hashObject.update(getOrCalculateHash(reference)); } const newHash = hashObject.digest('hex'); hashes[name] = newHash; @@ -107,8 +172,8 @@ function calculateHashes(fullyQualifiedNames, code) { }; return fullyQualifiedNames.map((fullyQualifiedName) => { - const name = `$author$project$${fullyQualifiedName.join('$')}`; - return { name: fullyQualifiedName, hash: getOrCalculateHash(name, [name]) }; + const name = fullyQualifiedNameToVariableName(fullyQualifiedName); + return { name: fullyQualifiedName, hash: getOrCalculateHash(name) }; }); } From 97ca1a9c2ae889a4b9e6331fcae69f89c8416b18 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Mon, 13 Jul 2026 22:43:57 +0200 Subject: [PATCH 03/81] Explode into several functions --- lib/Hash.js | 64 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index e1e994ae..d3a788f3 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -45,23 +45,11 @@ function fullyQualifiedNameToVariableName(fullyQualifiedName) { } /** - * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) - * which may be tests. For each fully qualified name, find the corresponding - * JavaScript definition in the compiled Elm JavaScript code, and return a - * hash of the code of that definition. If the definition refers to other - * definition (it calls other functions), the hash is based on both the hash - * of the code of the definition, and of the hashes of all referenced definitions. - * - * This way we can tell if the code that will be running via an exposed `Test` - * value has changed or not, and thus if we need to re-run it or not. - * - * @param { Array> } fullyQualifiedNames * @param { string } code - * @returns { Array<{ name: Array, hash: string }> } + * @returns { Record } */ -function calculateHashes(fullyQualifiedNames, code) { - /** @type { Record } */ - const chunks = Object.fromEntries( +function parseChunks(code) { + return Object.fromEntries( code.split(CHUNK_REGEX).flatMap((chunk) => { const match = CHUNK_DEFINITION_NAME_REGEX.exec(chunk); // Not all chunks contain a definition. @@ -72,11 +60,19 @@ function calculateHashes(fullyQualifiedNames, code) { return [[name, chunk]]; }) ); +} - /** - * @typedef { { code: Array, references: Set } } Item - * @type { Record } - */ +/** + * Resolves references for everything reachable from `fullyQualifiedNames` in `chunks`. + * Merges recursive chains into single items, so that the output is an acyclic graph. + * + * @typedef { { code: Array, references: Set } } Item + * @param { Array> } fullyQualifiedNames + * @param { Record } chunks + * @returns { Record } + */ +function toItems(fullyQualifiedNames, chunks) { + /** @type { Record } */ const items = {}; /** @@ -140,6 +136,15 @@ function calculateHashes(fullyQualifiedNames, code) { update(name, chunk, []); } + return items; +} + +/** + * @param { Array> } fullyQualifiedNames + * @param { Record } items + * @returns { Array<{ name: Array, hash: string }> } + */ +function doHashing(fullyQualifiedNames, items) { /** @type { Record } */ const hashes = {}; @@ -177,6 +182,27 @@ function calculateHashes(fullyQualifiedNames, code) { }); } +/** + * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) + * which may be tests. For each fully qualified name, find the corresponding + * JavaScript definition in the compiled Elm JavaScript code, and return a + * hash of the code of that definition. If the definition refers to other + * definitions (it calls other functions), the hash is based on both the hash + * of the code of the definition, and of the hashes of all referenced definitions. + * + * This way we can tell if the code that will be running via an exposed `Test` + * value has changed or not, and thus if we need to re-run it or not. + * + * @param { Array> } fullyQualifiedNames + * @param { string } code + * @returns { Array<{ name: Array, hash: string }> } + */ +function calculateHashes(fullyQualifiedNames, code) { + const chunks = parseChunks(code); + const items = toItems(fullyQualifiedNames, chunks); + return doHashing(fullyQualifiedNames, items); +} + module.exports = { calculateHashes, }; From d8853c4cfdebd8ebca6f06a98e545167e35b5d95 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Mon, 13 Jul 2026 23:09:00 +0200 Subject: [PATCH 04/81] Clean up --- lib/Hash.js | 144 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 87 insertions(+), 57 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index d3a788f3..f947f615 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -1,5 +1,26 @@ const crypto = require('crypto'); +/** + * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) + * which may be tests. For each fully qualified name, find the corresponding + * JavaScript definition in the compiled Elm JavaScript code, and return a + * hash of the code of that definition. If the definition refers to other + * definitions (it calls other functions), the hash is based on both the hash + * of the code of the definition, and of the hashes of all referenced definitions. + * + * This way we can tell if the code that will be running via an exposed `Test` + * value has changed or not, and thus if we need to re-run it or not. + * + * @param { Array> } fullyQualifiedNames + * @param { string } code + * @returns { Array<{ name: Array, hash: string }> } + */ +function calculateHashes(fullyQualifiedNames, code) { + const chunks = parseStep(code); + const graph = referencesStep(fullyQualifiedNames, chunks); + return hashStep(fullyQualifiedNames, graph); +} + /** * The compiled Elm JavaScript is basically just a long sequence of definitions. * Some are `function` statements, some are `var` assignments. @@ -45,10 +66,14 @@ function fullyQualifiedNameToVariableName(fullyQualifiedName) { } /** + * Splits `code` into chunks as defined by `CHUNK_REGEX`. + * Returns the chunks that contain a definition (variable or function), + * keyed by the definition name. + * * @param { string } code * @returns { Record } */ -function parseChunks(code) { +function parseStep(code) { return Object.fromEntries( code.split(CHUNK_REGEX).flatMap((chunk) => { const match = CHUNK_DEFINITION_NAME_REGEX.exec(chunk); @@ -66,62 +91,82 @@ function parseChunks(code) { * Resolves references for everything reachable from `fullyQualifiedNames` in `chunks`. * Merges recursive chains into single items, so that the output is an acyclic graph. * - * @typedef { { code: Array, references: Set } } Item + * @typedef { { + code: Array, + references: Set, + } } Node + * * @param { Array> } fullyQualifiedNames * @param { Record } chunks - * @returns { Record } + * @returns { Record } */ -function toItems(fullyQualifiedNames, chunks) { - /** @type { Record } */ - const items = {}; +function referencesStep(fullyQualifiedNames, chunks) { + /** @type { Record } */ + const graph = {}; /** * @param { string } name * @param { string } chunk - * @param { Array } seen + * @param { Array } seenPreviously * @returns { void } */ - const update = (name, chunk, seen) => { - if (name in items) { + const createNode = (name, chunk, seenPreviously) => { + // Already processed. + if (name in graph) { return; } + + // Create a node in the graph. const references = chunk.match(REFERENCES_REGEX); if (references === null) { throw new Error(`No references found in chunk for ${name}:\n${chunk}`); } - /** @type { Item } */ - const item = { + /** @type { Node } */ + const node = { code: [chunk], references: new Set( references.filter( (reference) => + // Skip string literals and comments and take only identifiers – see `REFERENCES_REGEX`. (reference.startsWith('$') || reference.startsWith('_')) && + // Skip direct recursion. reference !== name && + // Only care about references to stuff defined in `chunks`. reference in chunks ) ), }; - items[name] = item; - const newSeen = [...seen, name]; - for (const reference of item.references) { + graph[name] = node; + + // Create nodes in the graph for all references. + const seen = [...seenPreviously, name]; + for (const reference of node.references) { + // Note: We already checked `reference in chunks` when constructing `node.references`. const referenceChunk = chunks[reference]; - // TODO: Check if undefined or not? - let index = newSeen.indexOf(reference); + let index = seen.indexOf(reference); if (index === -1) { - update(reference, referenceChunk, newSeen); + createNode(reference, referenceChunk, seen); } else { - const chain = newSeen.slice(index); - /** @type { Item } */ - const newItem = { code: [], references: new Set() }; - for (const name2 of chain) { - const item2 = items[name2]; - newItem.code.push(...item2.code); - for (const n of item2.references) { - if (!chain.includes(n)) { - newItem.references.add(n); + // A chain of indirect recursion was found! + // Replace all the involved functions with the same node, + // containing the code and references of all the involved functions. + // This is how we make the graph acyclic. + const chain = seen.slice(index); + /** @type { Node } */ + const newNode = { + code: [], + references: new Set(), + }; + for (const chainName of chain) { + const chainNode = graph[chainName]; + newNode.code.push(...chainNode.code); + for (const chainReference of chainNode.references) { + // Skip direct recursion. + if (!chain.includes(chainReference)) { + newNode.references.add(chainReference); } } - items[name2] = newItem; + graph[chainName] = newNode; } } } @@ -133,18 +178,18 @@ function toItems(fullyQualifiedNames, chunks) { if (chunk === undefined) { throw new Error(`Could not find ${name} in the compiled code!`); } - update(name, chunk, []); + createNode(name, chunk, []); } - return items; + return graph; } /** * @param { Array> } fullyQualifiedNames - * @param { Record } items + * @param { Record } graph * @returns { Array<{ name: Array, hash: string }> } */ -function doHashing(fullyQualifiedNames, items) { +function hashStep(fullyQualifiedNames, graph) { /** @type { Record } */ const hashes = {}; @@ -153,24 +198,30 @@ function doHashing(fullyQualifiedNames, items) { * @returns { string } */ const getOrCalculateHash = (name) => { + // Already processed. const hash = hashes[name]; if (hash !== undefined) { return hash; } - const item = items[name]; - if (item === undefined) { - throw new Error(`No item for ${name}!`); + + const node = graph[name]; + if (node === undefined) { + throw new Error( + `Could not find ${name} in the graph of the compiled code!` + ); } + // When testing on a large project, all hashes led to about the same // amount of time used by `getOrCalculateHash`. `sha256` is one of // the ones being about 10 ms faster than the slowest ones. const hashObject = crypto.createHash('sha256'); - for (const code of Array.from(item.code).sort()) { + for (const code of Array.from(node.code).sort()) { hashObject.update(code); } - for (const reference of Array.from(item.references).sort()) { + for (const reference of Array.from(node.references).sort()) { hashObject.update(getOrCalculateHash(reference)); } + const newHash = hashObject.digest('hex'); hashes[name] = newHash; return newHash; @@ -182,27 +233,6 @@ function doHashing(fullyQualifiedNames, items) { }); } -/** - * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) - * which may be tests. For each fully qualified name, find the corresponding - * JavaScript definition in the compiled Elm JavaScript code, and return a - * hash of the code of that definition. If the definition refers to other - * definitions (it calls other functions), the hash is based on both the hash - * of the code of the definition, and of the hashes of all referenced definitions. - * - * This way we can tell if the code that will be running via an exposed `Test` - * value has changed or not, and thus if we need to re-run it or not. - * - * @param { Array> } fullyQualifiedNames - * @param { string } code - * @returns { Array<{ name: Array, hash: string }> } - */ -function calculateHashes(fullyQualifiedNames, code) { - const chunks = parseChunks(code); - const items = toItems(fullyQualifiedNames, chunks); - return doHashing(fullyQualifiedNames, items); -} - module.exports = { calculateHashes, }; From b55c2cf85afa88a769699bfd46113935807e63d2 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Mon, 13 Jul 2026 23:11:16 +0200 Subject: [PATCH 05/81] Helpful type alias --- lib/Hash.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index f947f615..6752d229 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -11,9 +11,11 @@ const crypto = require('crypto'); * This way we can tell if the code that will be running via an exposed `Test` * value has changed or not, and thus if we need to re-run it or not. * - * @param { Array> } fullyQualifiedNames + * @typedef { Array } FullyQualifiedName + * + * @param { Array } fullyQualifiedNames * @param { string } code - * @returns { Array<{ name: Array, hash: string }> } + * @returns { Array<{ name: FullyQualifiedName, hash: string }> } */ function calculateHashes(fullyQualifiedNames, code) { const chunks = parseStep(code); @@ -58,7 +60,7 @@ const REFERENCES_REGEX = /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$_\u200C\u200D\p{ID_Continue}]+/gu; /** - * @param { Array } fullyQualifiedName + * @param { FullyQualifiedName } fullyQualifiedName * @returns { string } */ function fullyQualifiedNameToVariableName(fullyQualifiedName) { @@ -96,7 +98,7 @@ function parseStep(code) { references: Set, } } Node * - * @param { Array> } fullyQualifiedNames + * @param { Array } fullyQualifiedNames * @param { Record } chunks * @returns { Record } */ @@ -185,9 +187,9 @@ function referencesStep(fullyQualifiedNames, chunks) { } /** - * @param { Array> } fullyQualifiedNames + * @param { Array } fullyQualifiedNames * @param { Record } graph - * @returns { Array<{ name: Array, hash: string }> } + * @returns { Array<{ name: FullyQualifiedName, hash: string }> } */ function hashStep(fullyQualifiedNames, graph) { /** @type { Record } */ From ddc651563e22e36a97a8e715e9f451fe0cd007be Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Mon, 13 Jul 2026 23:12:16 +0200 Subject: [PATCH 06/81] Avoid spread --- lib/Hash.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Hash.js b/lib/Hash.js index 6752d229..0cd112ed 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -161,7 +161,9 @@ function referencesStep(fullyQualifiedNames, chunks) { }; for (const chainName of chain) { const chainNode = graph[chainName]; - newNode.code.push(...chainNode.code); + for (const code of chainNode.code) { + newNode.code.push(code); + } for (const chainReference of chainNode.references) { // Skip direct recursion. if (!chain.includes(chainReference)) { From 9aa8630fe9a3399f6a131a1f8ca2476e380a6836 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 14 Jul 2026 18:48:56 +0200 Subject: [PATCH 07/81] Simplify API --- lib/Hash.js | 48 +++++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index 0cd112ed..c7a07e3d 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -1,8 +1,8 @@ const crypto = require('crypto'); /** - * Pass in an array of fully qualified names (such as `[['MyTest', 'suite']]`) - * which may be tests. For each fully qualified name, find the corresponding + * Pass in an array of definition names, such as `["$author$project$MyTest$suite"]`, + * which may be tests. For each definition name, find the corresponding * JavaScript definition in the compiled Elm JavaScript code, and return a * hash of the code of that definition. If the definition refers to other * definitions (it calls other functions), the hash is based on both the hash @@ -11,16 +11,14 @@ const crypto = require('crypto'); * This way we can tell if the code that will be running via an exposed `Test` * value has changed or not, and thus if we need to re-run it or not. * - * @typedef { Array } FullyQualifiedName - * - * @param { Array } fullyQualifiedNames + * @param { Array } names * @param { string } code - * @returns { Array<{ name: FullyQualifiedName, hash: string }> } + * @returns { Record } */ -function calculateHashes(fullyQualifiedNames, code) { +function calculateHashes(names, code) { const chunks = parseStep(code); - const graph = referencesStep(fullyQualifiedNames, chunks); - return hashStep(fullyQualifiedNames, graph); + const graph = referencesStep(names, chunks); + return hashStep(names, graph); } /** @@ -59,14 +57,6 @@ const CHUNK_DEFINITION_NAME_REGEX = /(?:var|function) ([^ (]+)/; const REFERENCES_REGEX = /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$_\u200C\u200D\p{ID_Continue}]+/gu; -/** - * @param { FullyQualifiedName } fullyQualifiedName - * @returns { string } - */ -function fullyQualifiedNameToVariableName(fullyQualifiedName) { - return `$author$project$${fullyQualifiedName.join('$')}`; -} - /** * Splits `code` into chunks as defined by `CHUNK_REGEX`. * Returns the chunks that contain a definition (variable or function), @@ -90,7 +80,7 @@ function parseStep(code) { } /** - * Resolves references for everything reachable from `fullyQualifiedNames` in `chunks`. + * Resolves references for everything reachable from `names` in `chunks`. * Merges recursive chains into single items, so that the output is an acyclic graph. * * @typedef { { @@ -98,11 +88,11 @@ function parseStep(code) { references: Set, } } Node * - * @param { Array } fullyQualifiedNames + * @param { Array } names * @param { Record } chunks * @returns { Record } */ -function referencesStep(fullyQualifiedNames, chunks) { +function referencesStep(names, chunks) { /** @type { Record } */ const graph = {}; @@ -176,8 +166,7 @@ function referencesStep(fullyQualifiedNames, chunks) { } }; - for (const fullyQualifiedName of fullyQualifiedNames) { - const name = fullyQualifiedNameToVariableName(fullyQualifiedName); + for (const name of names) { const chunk = chunks[name]; if (chunk === undefined) { throw new Error(`Could not find ${name} in the compiled code!`); @@ -189,11 +178,11 @@ function referencesStep(fullyQualifiedNames, chunks) { } /** - * @param { Array } fullyQualifiedNames + * @param { Array } names * @param { Record } graph - * @returns { Array<{ name: FullyQualifiedName, hash: string }> } + * @returns { Record } */ -function hashStep(fullyQualifiedNames, graph) { +function hashStep(names, graph) { /** @type { Record } */ const hashes = {}; @@ -231,10 +220,11 @@ function hashStep(fullyQualifiedNames, graph) { return newHash; }; - return fullyQualifiedNames.map((fullyQualifiedName) => { - const name = fullyQualifiedNameToVariableName(fullyQualifiedName); - return { name: fullyQualifiedName, hash: getOrCalculateHash(name) }; - }); + return Object.fromEntries( + names.map((name) => { + return [name, getOrCalculateHash(name)]; + }) + ); } module.exports = { From 8dbc8fce6283829e3b16fbdbde814242a815218d Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 14 Jul 2026 18:54:01 +0200 Subject: [PATCH 08/81] Use good old loops --- lib/Hash.js | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index c7a07e3d..d24f7c98 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -66,17 +66,17 @@ const REFERENCES_REGEX = * @returns { Record } */ function parseStep(code) { - return Object.fromEntries( - code.split(CHUNK_REGEX).flatMap((chunk) => { - const match = CHUNK_DEFINITION_NAME_REGEX.exec(chunk); - // Not all chunks contain a definition. - if (match === null) { - return []; - } + /** @type { Record } */ + const chunks = {}; + for (const chunk of code.split(CHUNK_REGEX)) { + const match = CHUNK_DEFINITION_NAME_REGEX.exec(chunk); + // Not all chunks contain a definition. + if (match !== null) { const name = match[1]; - return [[name, chunk]]; - }) - ); + chunks[name] = chunk; + } + } + return chunks; } /** @@ -220,11 +220,12 @@ function hashStep(names, graph) { return newHash; }; - return Object.fromEntries( - names.map((name) => { - return [name, getOrCalculateHash(name)]; - }) - ); + /** @type { Record } */ + const result = {}; + for (const name of names) { + result[name] = getOrCalculateHash(name); + } + return result; } module.exports = { From f15bd118499d7161c59f5de31623956de169ce55 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 14 Jul 2026 19:02:02 +0200 Subject: [PATCH 09/81] Comment --- lib/Hash.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/Hash.js b/lib/Hash.js index d24f7c98..d7f6aa81 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -150,6 +150,8 @@ function referencesStep(names, chunks) { references: new Set(), }; for (const chainName of chain) { + // Note: Since `chainName` is a name we have already visited, + // we know that we have inserted a node for it. const chainNode = graph[chainName]; for (const code of chainNode.code) { newNode.code.push(code); From 8247f8f5f862db988a26cc7bf18b9c17a6010374 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 14 Jul 2026 19:37:30 +0200 Subject: [PATCH 10/81] Optimize regex --- lib/Hash.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/Hash.js b/lib/Hash.js index d7f6aa81..49d8e68e 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -53,9 +53,13 @@ const CHUNK_DEFINITION_NAME_REGEX = /(?:var|function) ([^ (]+)/; * dollar sign or an underscore. We only care about out the identifiers, but match the * other literals too, so that we don’t get false positives for identifiers inside strings and comments. * Parts copied from: https://github.com/lydell/js-tokens/blob/895fb4d6804a287aecfb0e1009851f925d07b079/index.coffee + * A more exact regex for identifiers is `/[$_][$_\u200C\u200D\p{ID_Continue}]+/gu`, + * but the one we’re using is about twice as fast. We match ASCII identifier chars, + * and then _anything_ non-ASCII, because the only non-ASCII characters outside strings + * and comments are going to be identifiers. */ const REFERENCES_REGEX = - /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$_\u200C\u200D\p{ID_Continue}]+/gu; + /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$\w\u0080-\uffff]+/g; /** * Splits `code` into chunks as defined by `CHUNK_REGEX`. From e237e77ac3b32dae5062b4d6f3f280f73c86b5ee Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Wed, 15 Jul 2026 10:04:10 +0200 Subject: [PATCH 11/81] Make it all fit together but with TODOs --- elm/src/Test/Runner/Node.elm | 6 +- lib/Generate.js | 123 ++++++++++++++++++++++++++++++++--- lib/RunTests.js | 11 +++- 3 files changed, 126 insertions(+), 14 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 2e9f7cd8..2076599e 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -352,13 +352,13 @@ with a version that returns `Just value` if `value` is a `Test`, otherwise `Noth If you rename or change this function you also need to update the regex that looks for it. -} -check : a -> Maybe Test +check : a -> String -> String -> Maybe Test check = checkHelperReplaceMe___ -checkHelperReplaceMe___ : a -> b -checkHelperReplaceMe___ _ = +checkHelperReplaceMe___ : a -> String -> String -> b +checkHelperReplaceMe___ _ _ _ = Debug.todo "The regex for replacing this Debug.todo with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" diff --git a/lib/Generate.js b/lib/Generate.js index f38e5b0e..cdf1e129 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -2,8 +2,22 @@ const { supportsColor } = require('./chalk'); const fs = require('fs'); const path = require('path'); const ElmJson = require('./ElmJson'); +const Hash = require('./Hash'); const Solve = require('./Solve'); +// Stores all we need to know about a previous run to determine +// which tests haven’t changed and therefore can be skipped. +// TODO: Do we store test results? Or do we somehow only store the ones that passed? +const fingerprintsFileName = 'fingerprints.json'; + +/** + * @typedef { { + fuzz: number, + seed: number, + hashes: Record + } } Fingerprints + */ + const before = fs.readFileSync( path.join(__dirname, '..', 'templates', 'before.js'), 'utf8' @@ -15,21 +29,49 @@ const after = fs.readFileSync( ); /** + * @param { number } fuzz + * @param { number } seed + * @param { Array<{ + moduleName: string, + possiblyTests: Array, + }> } testModules * @param { string } pipeFilename * @param { string } dest * @returns { void } */ -function prepareCompiledJsFile(pipeFilename, dest) { +function prepareCompiledJsFile(fuzz, seed, testModules, pipeFilename, dest) { const content = fs.readFileSync(dest, 'utf8'); + + const names = testModules.flatMap((mod) => + mod.possiblyTests.map((test) => + toCompiledJavaScriptName(mod.moduleName, test) + ) + ); + + const hashes = Hash.calculateHashes(names, content); + + /** @type { Fingerprints } */ + const fingerprints = { + fuzz, + seed, + hashes, + }; + + fs.writeFileSync( + path.join(path.dirname(dest), fingerprintsFileName), + JSON.stringify(fingerprints) + ); + const finalContent = ` ${before} var Elm = (function(module) { -${addKernelTestChecking(content)} +${addKernelTestChecking(hashes, content)} return this.Elm; })({}); var pipeFilename = ${JSON.stringify(pipeFilename)}; ${after} `.trim(); + fs.writeFileSync(dest, finalContent); // Needed when the user has `"type": "module"` in their package.json. @@ -40,6 +82,32 @@ ${after} ); } +/** + * @param { string } dest + * @returns { Fingerprints } + */ +function readOldFingerprints(dest) { + try { + return JSON.parse( + fs.readFileSync( + path.join(path.dirname(dest), fingerprintsFileName), + 'utf-8' + ) + ); + } catch (error) { + if (error.code !== 'ENOENT') { + console.warn( + `Ignoring bad fingerprints file:\n\n${error.message}\n\nPlease report this issue: https://github.com/rtfeldman/node-test-runner/issues/new` + ); + } + } + return { + fuzz: -1, + seed: -1, + hashes: {}, + }; +} + // For older versions of elm-explorations/test we need to list every single // variant of the `Test` type. To avoid having to update this regex if a new // variant is added, newer versions of elm-explorations/test have prefixed all @@ -52,22 +120,45 @@ const testVariantDefinition = const checkDefinition = /^(var\s+\$author\$project\$Test\$Runner\$Node\$check)\s*=\s*\$author\$project\$Test\$Runner\$Node\$checkHelperReplaceMe___;?$/m; +// For the identifier, this uses the same regex as `REFERENCES_REGEX` in `Hash.js`. +const hashPlaceholder = /__elm_test_hash__:([$_][$\w\u0080-\uffff]+)/g; + +/** + * @param { string } name + * @returns { string } + */ +function makeHashPlaceholder(name) { + return `__elm_test_hash__:${name}`; +} + +/** + * @param { string } moduleName + * @param { string } valueName + * @returns { string } + */ +function toCompiledJavaScriptName(moduleName, valueName) { + return `$author$project$${moduleName.replace(/\./g, '$')}$${valueName}`; +} + /** * Create a symbol, tag all `Test` constructors with it and make the `check` * function look for it. * + * @param { Record } hashes * @param { string } content * @returns { string } */ -function addKernelTestChecking(content) { +function addKernelTestChecking(hashes, content) { return ( 'var __elmTestSymbol = Symbol("elmTestSymbol");\n' + content .replace(testVariantDefinition, '$&__elmTestSymbol: __elmTestSymbol, ') .replace( checkDefinition, - '$1 = value => value && value.__elmTestSymbol === __elmTestSymbol ? $elm$core$Maybe$Just(value) : $elm$core$Maybe$Nothing;' + // TODO: Also care about fuzz and seed, and don’t just filter them away. + '$1 = F3((value, oldHash, newHash) => value && value.__elmTestSymbol === __elmTestSymbol && oldHash !== newHash ? $elm$core$Maybe$Just(value) : $elm$core$Maybe$Nothing);' ) + .replace(hashPlaceholder, (_match, name) => hashes[name]) ); } @@ -160,6 +251,7 @@ function getMainModule(generatedCodeDir) { } /** + * @param { Fingerprints } oldFingerprints * @param { number } fuzz * @param { number } seed * @param { import('./Report').Report } report @@ -174,6 +266,7 @@ function getMainModule(generatedCodeDir) { * @returns { void } */ function generateMainModule( + oldFingerprints, fuzz, seed, report, @@ -184,6 +277,7 @@ function generateMainModule( processes ) { const testFileBody = makeTestFileBody( + oldFingerprints, testModules, makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths, processes) ); @@ -196,6 +290,7 @@ function generateMainModule( } /** + * @param { Fingerprints } oldFingerprints * @param { Array<{ moduleName: string, possiblyTests: Array, @@ -203,10 +298,12 @@ function generateMainModule( * @param { string } optsCode * @returns { string } */ -function makeTestFileBody(testModules, optsCode) { +function makeTestFileBody(oldFingerprints, testModules, optsCode) { const imports = testModules.map((mod) => `import ${mod.moduleName}`); - const possiblyTestsList = makeList(testModules.map(makeModuleTuple)); + const possiblyTestsList = makeList( + testModules.map((mod) => makeModuleTuple(oldFingerprints, mod)) + ); return ` ${imports.join('\n')} @@ -225,16 +322,21 @@ main = } /** + * @param { Fingerprints } oldFingerprints * @param { { moduleName: string, possiblyTests: Array, } } mod * @returns { string } */ -function makeModuleTuple(mod) { - const list = mod.possiblyTests.map( - (test) => `Test.Runner.Node.check ${mod.moduleName}.${test}` - ); +function makeModuleTuple(oldFingerprints, mod) { + const list = mod.possiblyTests.map((test) => { + const name = toCompiledJavaScriptName(mod.moduleName, test); + // TODO: Also care about fuzz and seed. + const oldHash = oldFingerprints.hashes[name] || ''; + const newHash = makeHashPlaceholder(name); + return `Test.Runner.Node.check ${mod.moduleName}.${test} "${oldHash}" "${newHash}"`; + }); return ` ( "${mod.moduleName}" @@ -340,4 +442,5 @@ module.exports = { generateMainModule, getMainModule, prepareCompiledJsFile, + readOldFingerprints, }; diff --git a/lib/RunTests.js b/lib/RunTests.js index 5d9b8af3..951d50d2 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -246,7 +246,10 @@ function runTests( progressLogger.log('Compiling'); + const oldFingerprints = Generate.readOldFingerprints(dest); + Generate.generateMainModule( + oldFingerprints, fuzz, seed, report, @@ -265,7 +268,13 @@ function runTests( report ); - Generate.prepareCompiledJsFile(pipeFilename, dest); + Generate.prepareCompiledJsFile( + fuzz, + seed, + testModules, + pipeFilename, + dest + ); progressLogger.log('Starting tests'); progressLogger.newLine(); From 00a2c96de186c2fccbb25a955021756bcf96b39f Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Wed, 15 Jul 2026 22:01:25 +0200 Subject: [PATCH 12/81] WIP try to save outcomes of each test --- elm/src/Test/Runner/Node.elm | 134 +++++++++++++++++++++++++++++++++-- lib/Generate.js | 32 +++++---- lib/RunTests.js | 1 - 3 files changed, 144 insertions(+), 23 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 2076599e..18a21e0c 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -13,6 +13,7 @@ passed and 2 if any failed. Returns 1 if something went wrong. -} import Dict exposing (Dict) +import Expect exposing (Expectation) import Json.Decode as Decode import Json.Encode as Encode import Platform @@ -42,6 +43,7 @@ type alias InitArgs = , fuzzRuns : Int , runners : SeededRunners , report : Report + , metadata : Metadata } @@ -63,9 +65,14 @@ type alias Model = , processes : Int , nextTestToRun : TestId , autoFail : Maybe String + , metadata : Metadata } +type alias Metadata = + Dict ( String, String ) { jsDefinitionName : String, hash : String } + + {-| A program which will run tests and report their results. -} type alias TestProgram = @@ -87,6 +94,27 @@ port elmTestPort__send : String -> Cmd msg port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg +type alias Fingerprints = + { hash : String + , outcomes : Dict ( String, String ) { isFuzzTest : Bool, expectations : List Expectation } + } + + +oldFuzzRuns : Int +oldFuzzRuns = + 0 + + +oldInitialSeed : Int +oldInitialSeed = + 0 + + +oldFingerprints : Dict String Fingerprints +oldFingerprints = + Dict.empty + + dispatch : Model -> Posix -> Cmd Msg dispatch model startTime = case Dict.get model.nextTestToRun model.available of @@ -96,13 +124,78 @@ dispatch model startTime = Just config -> let + maybeCachedExpectations = + lastTwoReversed config.labels + |> Maybe.andThen + (\key -> + Dict.get key model.metadata + |> Maybe.andThen + (\metadata -> + Dict.get metadata.jsDefinitionName oldFingerprints + |> Maybe.andThen + (\fingerprints -> + if metadata.hash == fingerprints.hash then + case Dict.get key fingerprints.outcomes of + Just outcome -> + if + not outcome.isFuzzTest + || ((model.runInfo.fuzzRuns <= oldFuzzRuns) + && (model.runInfo.initialSeed == oldInitialSeed) + ) + then + Just outcome.expectations + + else + Nothing + + Nothing -> + -- TODO: Supposed to construct a pass without distribution report here, but don’t know how + Just [] + + else + Nothing + ) + ) + ) + + expectations = + case maybeCachedExpectations of + Just expectations_ -> + expectations_ + + Nothing -> + config.run () + outcomes = - outcomesFromExpectations (config.run ()) + outcomesFromExpectations expectations in Time.now |> Task.perform (Complete config.labels outcomes startTime) +lastTwoReversed : List a -> Maybe ( a, a ) +lastTwoReversed list = + case list of + [ a, b ] -> + Just ( b, a ) + + _ :: rest -> + lastTwoReversed rest + + _ -> + Nothing + + +runTest config = + -- Replace with kernel code that: + -- looks up previous data + -- check hashes + -- if previous data says fuzz test, also check fuzz and seed + -- if ok, use previous expectations (Or Passed NoDistribution if no saved file) + -- if not ok, run tests and save outcome + config.run () + + update : Msg -> Model -> ( Model, Cmd Msg ) update msg ({ testReporter } as model) = case msg of @@ -266,7 +359,7 @@ sendBegin model = init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { processes, globs, paths, fuzzRuns, initialSeed, report, runners } _ = +init { processes, globs, paths, fuzzRuns, initialSeed, report, runners, metadata } _ = let { indexedRunners, autoFail } = case runners of @@ -296,6 +389,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, runners } _ = testReporter = createReporter report + model : Model model = { available = Dict.fromList indexedRunners , runInfo = @@ -310,6 +404,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, runners } _ = , results = [] , testReporter = testReporter , autoFail = autoFail + , metadata = metadata } in ( model, Cmd.none ) @@ -318,6 +413,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, runners } _ = failInit : String -> Report -> Int -> ( Model, Cmd Msg ) failInit message report _ = let + model : Model model = { available = Dict.empty , runInfo = @@ -332,6 +428,7 @@ failInit message report _ = , results = [] , testReporter = createReporter report , autoFail = Nothing + , metadata = Dict.empty } cmd = @@ -346,13 +443,21 @@ failInit message report _ = ( model, cmd ) +type alias TestWithMetadata = + { test : Test + , jsDefinitionName : String + , hash : String + , label : String + } + + {-| The implementation of this function will be replaced in the generated JS with a version that returns `Just value` if `value` is a `Test`, otherwise `Nothing`. If you rename or change this function you also need to update the regex that looks for it. -} -check : a -> String -> String -> Maybe Test +check : a -> String -> String -> Maybe TestWithMetadata check = checkHelperReplaceMe___ @@ -364,23 +469,37 @@ checkHelperReplaceMe___ _ _ _ = {-| Run the tests. -} -run : RunnerOptions -> List ( String, List (Maybe Test) ) -> Program Int Model Msg +run : RunnerOptions -> List ( String, List (Maybe TestWithMetadata) ) -> Program Int Model Msg run { runs, seed, report, globs, paths, processes } possiblyTests = let - tests = + ( tests, metadata ) = possiblyTests |> List.filterMap (\( moduleName, maybeModuleTests ) -> let - moduleTests = + moduleTestsWithMetadata = List.filterMap identity maybeModuleTests + + moduleTests = + List.map .test moduleTestsWithMetadata in if List.isEmpty moduleTests then Nothing else - Just (Test.describe moduleName moduleTests) + Just + ( Test.describe moduleName moduleTests + , moduleTestsWithMetadata + |> List.map + (\data -> + ( ( moduleName, data.label ) + , { jsDefinitionName = data.jsDefinitionName, hash = data.hash } + ) + ) + ) ) + |> List.unzip + |> Tuple.mapSecond (List.concat >> Dict.fromList) in if List.isEmpty tests then Platform.worker @@ -403,6 +522,7 @@ run { runs, seed, report, globs, paths, processes } possiblyTests = , fuzzRuns = runs , runners = runners , report = report + , metadata = metadata } in Platform.worker diff --git a/lib/Generate.js b/lib/Generate.js index cdf1e129..90b23426 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -155,8 +155,19 @@ function addKernelTestChecking(hashes, content) { .replace(testVariantDefinition, '$&__elmTestSymbol: __elmTestSymbol, ') .replace( checkDefinition, - // TODO: Also care about fuzz and seed, and don’t just filter them away. - '$1 = F3((value, oldHash, newHash) => value && value.__elmTestSymbol === __elmTestSymbol && oldHash !== newHash ? $elm$core$Maybe$Just(value) : $elm$core$Maybe$Nothing);' + ` +function _getElmTestLabel(test) { + while (typeof test.a !== "string" && typeof test.a !== undefined) { + test = test.a; + } + return test.a || ""; +} +$1 = F3((test, jsDefinitionName, hash) => + test && test.__elmTestSymbol === __elmTestSymbol + ? $elm$core$Maybe$Just({ test, jsDefinitionName, hash, label: _getElmTestLabel(test) }) + : $elm$core$Maybe$Nothing +); +`.trim() ) .replace(hashPlaceholder, (_match, name) => hashes[name]) ); @@ -251,7 +262,6 @@ function getMainModule(generatedCodeDir) { } /** - * @param { Fingerprints } oldFingerprints * @param { number } fuzz * @param { number } seed * @param { import('./Report').Report } report @@ -266,7 +276,6 @@ function getMainModule(generatedCodeDir) { * @returns { void } */ function generateMainModule( - oldFingerprints, fuzz, seed, report, @@ -277,7 +286,6 @@ function generateMainModule( processes ) { const testFileBody = makeTestFileBody( - oldFingerprints, testModules, makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths, processes) ); @@ -290,7 +298,6 @@ function generateMainModule( } /** - * @param { Fingerprints } oldFingerprints * @param { Array<{ moduleName: string, possiblyTests: Array, @@ -298,12 +305,10 @@ function generateMainModule( * @param { string } optsCode * @returns { string } */ -function makeTestFileBody(oldFingerprints, testModules, optsCode) { +function makeTestFileBody(testModules, optsCode) { const imports = testModules.map((mod) => `import ${mod.moduleName}`); - const possiblyTestsList = makeList( - testModules.map((mod) => makeModuleTuple(oldFingerprints, mod)) - ); + const possiblyTestsList = makeList(testModules.map(makeModuleTuple)); return ` ${imports.join('\n')} @@ -322,20 +327,17 @@ main = } /** - * @param { Fingerprints } oldFingerprints * @param { { moduleName: string, possiblyTests: Array, } } mod * @returns { string } */ -function makeModuleTuple(oldFingerprints, mod) { +function makeModuleTuple(mod) { const list = mod.possiblyTests.map((test) => { const name = toCompiledJavaScriptName(mod.moduleName, test); - // TODO: Also care about fuzz and seed. - const oldHash = oldFingerprints.hashes[name] || ''; const newHash = makeHashPlaceholder(name); - return `Test.Runner.Node.check ${mod.moduleName}.${test} "${oldHash}" "${newHash}"`; + return `Test.Runner.Node.check ${mod.moduleName}.${test} "${name}" "${newHash}"`; }); return ` diff --git a/lib/RunTests.js b/lib/RunTests.js index 951d50d2..ca78b01f 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -249,7 +249,6 @@ function runTests( const oldFingerprints = Generate.readOldFingerprints(dest); Generate.generateMainModule( - oldFingerprints, fuzz, seed, report, From b8d667de5f454335e5f950cd259405dd1a1da589 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Wed, 15 Jul 2026 22:04:53 +0200 Subject: [PATCH 13/81] Store outcomes instead --- elm/src/Test/Runner/Node.elm | 47 ++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 18a21e0c..b29e38c9 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -96,7 +96,7 @@ port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg type alias Fingerprints = { hash : String - , outcomes : Dict ( String, String ) { isFuzzTest : Bool, expectations : List Expectation } + , outcomes : Dict ( String, String ) { isFuzzTest : Bool, outcomes : List Outcome } } @@ -124,7 +124,7 @@ dispatch model startTime = Just config -> let - maybeCachedExpectations = + maybeCachedOutcomes = lastTwoReversed config.labels |> Maybe.andThen (\key -> @@ -135,22 +135,20 @@ dispatch model startTime = |> Maybe.andThen (\fingerprints -> if metadata.hash == fingerprints.hash then - case Dict.get key fingerprints.outcomes of - Just outcome -> - if - not outcome.isFuzzTest - || ((model.runInfo.fuzzRuns <= oldFuzzRuns) - && (model.runInfo.initialSeed == oldInitialSeed) - ) - then - Just outcome.expectations - - else - Nothing - - Nothing -> - -- TODO: Supposed to construct a pass without distribution report here, but don’t know how - Just [] + Dict.get key fingerprints.outcomes + |> Maybe.andThen + (\outcome -> + if + not outcome.isFuzzTest + || ((model.runInfo.fuzzRuns <= oldFuzzRuns) + && (model.runInfo.initialSeed == oldInitialSeed) + ) + then + Just outcome.outcomes + + else + Nothing + ) else Nothing @@ -158,16 +156,13 @@ dispatch model startTime = ) ) - expectations = - case maybeCachedExpectations of - Just expectations_ -> - expectations_ + outcomes = + case maybeCachedOutcomes of + Just outcomes_ -> + outcomes_ Nothing -> - config.run () - - outcomes = - outcomesFromExpectations expectations + outcomesFromExpectations (config.run ()) in Time.now |> Task.perform (Complete config.labels outcomes startTime) From 83b5411660b9b6bffdf37fb14e09eb26f3c7bf70 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Wed, 15 Jul 2026 22:09:06 +0200 Subject: [PATCH 14/81] Write down some plans --- elm/src/Test/Runner/Node.elm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index b29e38c9..e18f4f0c 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -112,6 +112,14 @@ oldInitialSeed = oldFingerprints : Dict String Fingerprints oldFingerprints = + -- Have these three definitions in a separate file? + -- Could import a file with the hardcoded empty values and exposing (..) + -- then insert the real values in this file, they shadow the imports + -- Note: Can code-gen easily with Debug.toString + -- Update `sendResults` to also send what we need to build the file (Debug.toString-ed stuff) + -- When tests done, assemble everything we need + -- Also detect fuzz test. Patch fuzzLoop, make a wrapper function around `config.run ()` + -- that resets the global and reads it and returns Dict.empty From a84beec76d70214f9935d83a90b1f8e4f15ef925 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 09:13:41 +0200 Subject: [PATCH 15/81] Fix outcomes key --- elm/src/Test/Runner/Node.elm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index e18f4f0c..13e97a9a 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -96,7 +96,7 @@ port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg type alias Fingerprints = { hash : String - , outcomes : Dict ( String, String ) { isFuzzTest : Bool, outcomes : List Outcome } + , outcomes : Dict (List String) { isFuzzTest : Bool, outcomes : List Outcome } } @@ -143,7 +143,7 @@ dispatch model startTime = |> Maybe.andThen (\fingerprints -> if metadata.hash == fingerprints.hash then - Dict.get key fingerprints.outcomes + Dict.get config.labels fingerprints.outcomes |> Maybe.andThen (\outcome -> if From 91f36537c66ff2a924c4cc3cab7324296cf59d58 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 09:24:01 +0200 Subject: [PATCH 16/81] Prepare for saving outcomes --- elm/src/Test/Runner/Node.elm | 57 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 13e97a9a..be524504 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -82,7 +82,7 @@ type alias TestProgram = type Msg = Receive Decode.Value | Dispatch Posix - | Complete (List String) (List Outcome) Posix Posix + | Complete (List String) Outcome2 Posix Posix {-| The port names are prefixed to reduce the likelihood of the project @@ -96,7 +96,13 @@ port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg type alias Fingerprints = { hash : String - , outcomes : Dict (List String) { isFuzzTest : Bool, outcomes : List Outcome } + , outcomes : Dict (List String) Outcome2 + } + + +type alias Outcome2 = + { isFuzzTest : Bool + , outcomes : List Outcome } @@ -112,14 +118,13 @@ oldInitialSeed = oldFingerprints : Dict String Fingerprints oldFingerprints = - -- Have these three definitions in a separate file? + -- TODO: + -- Have these three definitions in a separate file? Or pass around? -- Could import a file with the hardcoded empty values and exposing (..) -- then insert the real values in this file, they shadow the imports -- Note: Can code-gen easily with Debug.toString -- Update `sendResults` to also send what we need to build the file (Debug.toString-ed stuff) -- When tests done, assemble everything we need - -- Also detect fuzz test. Patch fuzzLoop, make a wrapper function around `config.run ()` - -- that resets the global and reads it and returns Dict.empty @@ -132,7 +137,7 @@ dispatch model startTime = Just config -> let - maybeCachedOutcomes = + maybeCachedOutcome = lastTwoReversed config.labels |> Maybe.andThen (\key -> @@ -145,14 +150,14 @@ dispatch model startTime = if metadata.hash == fingerprints.hash then Dict.get config.labels fingerprints.outcomes |> Maybe.andThen - (\outcome -> + (\outcome_ -> if - not outcome.isFuzzTest + not outcome_.isFuzzTest || ((model.runInfo.fuzzRuns <= oldFuzzRuns) && (model.runInfo.initialSeed == oldInitialSeed) ) then - Just outcome.outcomes + Just outcome_ else Nothing @@ -164,16 +169,16 @@ dispatch model startTime = ) ) - outcomes = - case maybeCachedOutcomes of - Just outcomes_ -> - outcomes_ + outcome = + case maybeCachedOutcome of + Just outcome_ -> + outcome_ Nothing -> - outcomesFromExpectations (config.run ()) + runTestAndCheckIfFuzzTest config.run in Time.now - |> Task.perform (Complete config.labels outcomes startTime) + |> Task.perform (Complete config.labels outcome startTime) lastTwoReversed : List a -> Maybe ( a, a ) @@ -189,14 +194,15 @@ lastTwoReversed list = Nothing -runTest config = +runTestAndCheckIfFuzzTest : (() -> List Expectation) -> Outcome2 +runTestAndCheckIfFuzzTest run_ = -- Replace with kernel code that: - -- looks up previous data - -- check hashes - -- if previous data says fuzz test, also check fuzz and seed - -- if ok, use previous expectations (Or Passed NoDistribution if no saved file) - -- if not ok, run tests and save outcome - config.run () + -- Resets global `isFuzzTest` var to `false` + -- Reads it again instead of `False` below + -- + patch `fuzzLoop` to set `isFuzzTest` to `true` + { outcomes = outcomesFromExpectations (run_ ()) + , isFuzzTest = False + } update : Msg -> Model -> ( Model, Cmd Msg ) @@ -269,7 +275,8 @@ update msg ({ testReporter } as model) = Dispatch startTime -> ( model, dispatch model startTime ) - Complete labels outcomes startTime endTime -> + -- TODO: Also use .isFuzzTest + Complete labels outcome2 startTime endTime -> let duration = Time.posixToMillis endTime - Time.posixToMillis startTime @@ -281,7 +288,7 @@ update msg ({ testReporter } as model) = :: rest results = - List.foldl prependOutcome model.results outcomes + List.foldl prependOutcome model.results outcome2.outcomes nextTestToRun = model.nextTestToRun + model.processes @@ -289,7 +296,7 @@ update msg ({ testReporter } as model) = isFinished = nextTestToRun >= model.runInfo.testCount in - if isFinished || List.any isFailure outcomes then + if isFinished || List.any isFailure outcome2.outcomes then let cmd = sendResults isFinished testReporter results From 9712020b308d2f99d5e5121c02e8e53a7acd865e Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 09:32:11 +0200 Subject: [PATCH 17/81] Pass along the metadata --- elm/src/Test/Runner/Node.elm | 74 ++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index be524504..b4704dab 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -70,7 +70,13 @@ type alias Model = type alias Metadata = - Dict ( String, String ) { jsDefinitionName : String, hash : String } + Dict ( String, String ) MetadataItem + + +type alias MetadataItem = + { jsDefinitionName : String + , hash : String + } {-| A program which will run tests and report their results. @@ -82,7 +88,7 @@ type alias TestProgram = type Msg = Receive Decode.Value | Dispatch Posix - | Complete (List String) Outcome2 Posix Posix + | Complete MetadataItem (List String) Outcome2 Posix Posix {-| The port names are prefixed to reduce the likelihood of the project @@ -137,36 +143,38 @@ dispatch model startTime = Just config -> let + metadata = + case lastTwoReversed config.labels |> Maybe.andThen (\key -> Dict.get key model.metadata) of + Just metadata_ -> + metadata_ + + -- This should not happen: All tests should have metadata. + -- TODO: Can we get here for `Test.todo`? + Nothing -> + { jsDefinitionName = "MISSING:" ++ Debug.toString config.labels, hash = "" } + maybeCachedOutcome = - lastTwoReversed config.labels + Dict.get metadata.jsDefinitionName oldFingerprints |> Maybe.andThen - (\key -> - Dict.get key model.metadata - |> Maybe.andThen - (\metadata -> - Dict.get metadata.jsDefinitionName oldFingerprints - |> Maybe.andThen - (\fingerprints -> - if metadata.hash == fingerprints.hash then - Dict.get config.labels fingerprints.outcomes - |> Maybe.andThen - (\outcome_ -> - if - not outcome_.isFuzzTest - || ((model.runInfo.fuzzRuns <= oldFuzzRuns) - && (model.runInfo.initialSeed == oldInitialSeed) - ) - then - Just outcome_ - - else - Nothing - ) - - else - Nothing - ) - ) + (\fingerprints -> + if metadata.hash == fingerprints.hash then + Dict.get config.labels fingerprints.outcomes + |> Maybe.andThen + (\outcome_ -> + if + not outcome_.isFuzzTest + || ((model.runInfo.fuzzRuns <= oldFuzzRuns) + && (model.runInfo.initialSeed == oldInitialSeed) + ) + then + Just outcome_ + + else + Nothing + ) + + else + Nothing ) outcome = @@ -178,7 +186,7 @@ dispatch model startTime = runTestAndCheckIfFuzzTest config.run in Time.now - |> Task.perform (Complete config.labels outcome startTime) + |> Task.perform (Complete metadata config.labels outcome startTime) lastTwoReversed : List a -> Maybe ( a, a ) @@ -275,8 +283,8 @@ update msg ({ testReporter } as model) = Dispatch startTime -> ( model, dispatch model startTime ) - -- TODO: Also use .isFuzzTest - Complete labels outcome2 startTime endTime -> + -- TODO: Also use metadata and .isFuzzTest + Complete metadata labels outcome2 startTime endTime -> let duration = Time.posixToMillis endTime - Time.posixToMillis startTime From f11452d4993b370c489a922687e6643956cadce8 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 10:08:53 +0200 Subject: [PATCH 18/81] Send new stuff back to the supervisor --- elm/src/Test/Reporter/JUnit.elm | 8 +++++++- elm/src/Test/Reporter/TestResults.elm | 2 ++ elm/src/Test/Runner/Node.elm | 28 +++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/elm/src/Test/Reporter/JUnit.elm b/elm/src/Test/Reporter/JUnit.elm index a28c4824..07c97f6e 100644 --- a/elm/src/Test/Reporter/JUnit.elm +++ b/elm/src/Test/Reporter/JUnit.elm @@ -128,7 +128,13 @@ reportComplete { labels, duration, outcome } = encodeExtraFailure : String -> Value encodeExtraFailure _ = - reportComplete { labels = [], duration = 0, outcome = Failed [] } + reportComplete + { labels = [] + , duration = 0 + , outcome = Failed [] + , jsDefinitionName = "" + , isFuzzTest = False + } reportSummary : SummaryInfo -> Maybe String -> Value diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index eeb241c3..41e5acc1 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -23,6 +23,8 @@ type alias TestResult = { labels : List String , outcome : Outcome , duration : Int -- in milliseconds + , jsDefinitionName : String + , isFuzzTest : Bool } diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index b4704dab..58ae9b1c 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -283,15 +283,27 @@ update msg ({ testReporter } as model) = Dispatch startTime -> ( model, dispatch model startTime ) - -- TODO: Also use metadata and .isFuzzTest Complete metadata labels outcome2 startTime endTime -> let duration = Time.posixToMillis endTime - Time.posixToMillis startTime prependOutcome outcome rest = + -- NOTE: This can add multiple results with the same test ID. + -- Later in `sendResults` we encode the results as a JSON object + -- keyed by test ID. When parsing that JSON, the last one of + -- each duplicate key wins. All in all, the code gives the + -- impression of that a single test somehow can result in multiple + -- outcomes, and for a while the code supports that, but then we + -- implicitly decided there is a single outcome and forget about + -- the rest. If there ever were any. I’m not sure. ( model.nextTestToRun - , { labels = labels, outcome = outcome, duration = duration } + , { labels = labels + , outcome = outcome + , duration = duration + , jsDefinitionName = metadata.jsDefinitionName + , isFuzzTest = outcome2.isFuzzTest + } ) :: rest @@ -342,6 +354,17 @@ sendResults isFinished testReporter results = -- These are coming in in reverse order. Doing a foldl with :: -- means we reverse the list again, while also doing the conversion! ( String.fromInt testId, testReporter.reportComplete result ) :: list + + encodeNewStuff ( _, result ) = + let + dictTuple : ( List String, Outcome2 ) + dictTuple = + ( result.labels, { isFuzzTest = result.isFuzzTest, outcomes = [ result.outcome ] } ) + in + Encode.object + [ ( "jsDefinitionName", Encode.string result.jsDefinitionName ) + , ( "dictTupleElmCode", Encode.string (Debug.toString dictTuple) ) + ] in Encode.object [ ( "type", Encode.string typeStr ) @@ -350,6 +373,7 @@ sendResults isFinished testReporter results = |> List.foldl addToKeyValues [] |> Encode.object ) + , ( "newStuff", Encode.list encodeNewStuff results ) ] |> Encode.encode 0 |> elmTestPort__send From 6b291a1054b8512c4570363c5901b0003b778fda Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 10:34:18 +0200 Subject: [PATCH 19/81] Require a single expectation --- elm/src/Test/Reporter/TestResults.elm | 79 +++++---------------------- elm/src/Test/Runner/Node.elm | 27 +++------ 2 files changed, 23 insertions(+), 83 deletions(-) diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index 41e5acc1..2f4be226 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -4,7 +4,7 @@ module Test.Reporter.TestResults exposing , SummaryInfo , TestResult , isFailure - , outcomesFromExpectations + , outcomeFromExpectations ) import Expect exposing (Expectation) @@ -54,75 +54,26 @@ isFailure outcome = False -outcomesFromExpectations : List Expectation -> List Outcome -outcomesFromExpectations expectations = +outcomeFromExpectations : List Expectation -> Outcome +outcomeFromExpectations expectations = case expectations of - expectation :: [] -> - -- Most often we'll get exactly 1 pass, so try that case first! + -- The type of test runner functions says that they return `List Expectation`, + -- but in practice they only ever return lists with exactly one item: + -- https://github.com/elm-explorations/test/pull/244 + -- That PR was reverted because it unfortunately was a breaking change for the package: + -- https://github.com/elm-explorations/test/commit/11f70d5fc0b6fdc88d7a34ea1d10f56969890493 + -- But to keep things simpler here, we only support exactly one expectation. + [ expectation ] -> case Test.Runner.getFailureReason expectation of Nothing -> - [ Passed (Test.Runner.getDistributionReport expectation) ] + Passed (Test.Runner.getDistributionReport expectation) Just failure -> if Test.Runner.isTodo expectation then - [ Todo failure.description ] + Todo failure.description else - [ Failed - [ ( failure, Test.Runner.getDistributionReport expectation ) ] - ] - - _ :: _ -> - let - builder = - List.foldl outcomesFromExpectationsHelp - { passes = [], todos = [], failures = [] } - expectations - - failuresList = - case builder.failures of - [] -> - [] - - failures -> - [ Failed failures ] - in - List.concat - [ List.map Passed builder.passes - , List.map Todo builder.todos - , failuresList - ] - - [] -> - [] - - -type alias OutcomeBuilder = - { passes : List DistributionReport - , todos : List String - , failures : List ( Failure, DistributionReport ) - } - + Failed [ ( failure, Test.Runner.getDistributionReport expectation ) ] -outcomesFromExpectationsHelp : Expectation -> OutcomeBuilder -> OutcomeBuilder -outcomesFromExpectationsHelp expectation builder = - case Test.Runner.getFailureReason expectation of - Just failure -> - if Test.Runner.isTodo expectation then - { builder | todos = failure.description :: builder.todos } - - else - { builder - | failures = - ( failure - , Test.Runner.getDistributionReport expectation - ) - :: builder.failures - } - - Nothing -> - { builder - | passes = - Test.Runner.getDistributionReport expectation - :: builder.passes - } + _ -> + Debug.todo ("A test somehow did not return exactly 1 expectation, it returned " ++ String.fromInt (List.length expectations) ++ "!") diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 58ae9b1c..fa9183a6 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -21,7 +21,7 @@ import Random import Task import Test exposing (Test) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) -import Test.Reporter.TestResults exposing (Outcome, TestResult, isFailure, outcomesFromExpectations) +import Test.Reporter.TestResults exposing (Outcome, TestResult, isFailure, outcomeFromExpectations) import Test.Runner exposing (Runner, SeededRunners(..)) import Test.Runner.JsMessage as JsMessage exposing (JsMessage(..)) import Time exposing (Posix) @@ -108,7 +108,7 @@ type alias Fingerprints = type alias Outcome2 = { isFuzzTest : Bool - , outcomes : List Outcome + , outcome : Outcome } @@ -208,7 +208,7 @@ runTestAndCheckIfFuzzTest run_ = -- Resets global `isFuzzTest` var to `false` -- Reads it again instead of `False` below -- + patch `fuzzLoop` to set `isFuzzTest` to `true` - { outcomes = outcomesFromExpectations (run_ ()) + { outcome = outcomeFromExpectations (run_ ()) , isFuzzTest = False } @@ -288,27 +288,16 @@ update msg ({ testReporter } as model) = duration = Time.posixToMillis endTime - Time.posixToMillis startTime - prependOutcome outcome rest = - -- NOTE: This can add multiple results with the same test ID. - -- Later in `sendResults` we encode the results as a JSON object - -- keyed by test ID. When parsing that JSON, the last one of - -- each duplicate key wins. All in all, the code gives the - -- impression of that a single test somehow can result in multiple - -- outcomes, and for a while the code supports that, but then we - -- implicitly decided there is a single outcome and forget about - -- the rest. If there ever were any. I’m not sure. + results = ( model.nextTestToRun , { labels = labels - , outcome = outcome + , outcome = outcome2.outcome , duration = duration , jsDefinitionName = metadata.jsDefinitionName , isFuzzTest = outcome2.isFuzzTest } ) - :: rest - - results = - List.foldl prependOutcome model.results outcome2.outcomes + :: model.results nextTestToRun = model.nextTestToRun + model.processes @@ -316,7 +305,7 @@ update msg ({ testReporter } as model) = isFinished = nextTestToRun >= model.runInfo.testCount in - if isFinished || List.any isFailure outcome2.outcomes then + if isFinished || isFailure outcome2.outcome then let cmd = sendResults isFinished testReporter results @@ -359,7 +348,7 @@ sendResults isFinished testReporter results = let dictTuple : ( List String, Outcome2 ) dictTuple = - ( result.labels, { isFuzzTest = result.isFuzzTest, outcomes = [ result.outcome ] } ) + ( result.labels, { isFuzzTest = result.isFuzzTest, outcome = result.outcome } ) in Encode.object [ ( "jsDefinitionName", Encode.string result.jsDefinitionName ) From be840397d95a9813e0f926b1fc7909cd677ca778 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 11:07:06 +0200 Subject: [PATCH 20/81] Move previous run stuff to model --- elm/src/Test/Runner/Node.elm | 51 +++++++++++++++++------------------- lib/Generate.js | 7 +++++ 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index fa9183a6..21a2e887 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -44,6 +44,7 @@ type alias InitArgs = , runners : SeededRunners , report : Report , metadata : Metadata + , previousRun : PreviousRun } @@ -54,6 +55,7 @@ type alias RunnerOptions = , globs : List String , paths : List String , processes : Int + , previousRun : PreviousRun } @@ -66,6 +68,7 @@ type alias Model = , nextTestToRun : TestId , autoFail : Maybe String , metadata : Metadata + , previousRun : PreviousRun } @@ -79,6 +82,13 @@ type alias MetadataItem = } +type alias PreviousRun = + { fuzzRuns : Int + , initialSeed : Int + , fingerprints : Dict String Fingerprints + } + + {-| A program which will run tests and report their results. -} type alias TestProgram = @@ -112,28 +122,6 @@ type alias Outcome2 = } -oldFuzzRuns : Int -oldFuzzRuns = - 0 - - -oldInitialSeed : Int -oldInitialSeed = - 0 - - -oldFingerprints : Dict String Fingerprints -oldFingerprints = - -- TODO: - -- Have these three definitions in a separate file? Or pass around? - -- Could import a file with the hardcoded empty values and exposing (..) - -- then insert the real values in this file, they shadow the imports - -- Note: Can code-gen easily with Debug.toString - -- Update `sendResults` to also send what we need to build the file (Debug.toString-ed stuff) - -- When tests done, assemble everything we need - Dict.empty - - dispatch : Model -> Posix -> Cmd Msg dispatch model startTime = case Dict.get model.nextTestToRun model.available of @@ -154,7 +142,7 @@ dispatch model startTime = { jsDefinitionName = "MISSING:" ++ Debug.toString config.labels, hash = "" } maybeCachedOutcome = - Dict.get metadata.jsDefinitionName oldFingerprints + Dict.get metadata.jsDefinitionName model.previousRun.fingerprints |> Maybe.andThen (\fingerprints -> if metadata.hash == fingerprints.hash then @@ -163,8 +151,8 @@ dispatch model startTime = (\outcome_ -> if not outcome_.isFuzzTest - || ((model.runInfo.fuzzRuns <= oldFuzzRuns) - && (model.runInfo.initialSeed == oldInitialSeed) + || ((model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) + && (model.runInfo.initialSeed == model.previousRun.initialSeed) ) then Just outcome_ @@ -362,6 +350,8 @@ sendResults isFinished testReporter results = |> List.foldl addToKeyValues [] |> Encode.object ) + + -- TODO: Actually care about this in Supervisor , ( "newStuff", Encode.list encodeNewStuff results ) ] |> Encode.encode 0 @@ -390,7 +380,7 @@ sendBegin model = init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { processes, globs, paths, fuzzRuns, initialSeed, report, runners, metadata } _ = +init { processes, globs, paths, fuzzRuns, initialSeed, report, runners, metadata, previousRun } _ = let { indexedRunners, autoFail } = case runners of @@ -436,6 +426,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, runners, metadata , testReporter = testReporter , autoFail = autoFail , metadata = metadata + , previousRun = previousRun } in ( model, Cmd.none ) @@ -460,6 +451,11 @@ failInit message report _ = , testReporter = createReporter report , autoFail = Nothing , metadata = Dict.empty + , previousRun = + { fuzzRuns = 0 + , initialSeed = 0 + , fingerprints = Dict.empty + } } cmd = @@ -501,7 +497,7 @@ checkHelperReplaceMe___ _ _ _ = {-| Run the tests. -} run : RunnerOptions -> List ( String, List (Maybe TestWithMetadata) ) -> Program Int Model Msg -run { runs, seed, report, globs, paths, processes } possiblyTests = +run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = let ( tests, metadata ) = possiblyTests @@ -554,6 +550,7 @@ run { runs, seed, report, globs, paths, processes } possiblyTests = , runners = runners , report = report , metadata = metadata + , previousRun = previousRun } in Platform.worker diff --git a/lib/Generate.js b/lib/Generate.js index 90b23426..b1737e9d 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -313,6 +313,7 @@ function makeTestFileBody(testModules, optsCode) { return ` ${imports.join('\n')} +import Dict import Test.Reporter.Reporter exposing (Report(..)) import Console.Text exposing (UseColor(..)) import Test.Runner.Node @@ -405,6 +406,12 @@ function makeOptsCode( ${indentAllButFirstLine(' ', makeList(testFileGlobs.map(makeElmString)))} , paths = ${indentAllButFirstLine(' ', makeList(testFilePaths.map(makeElmString)))} +-- TODO: Pass actual values here +, previousRun = + { fuzzRuns = 0 + , initialSeed = 0 + , fingerprints = Dict.empty + } } `.trim(); } From c1e2a85be86cea4c251bec13d21bbb4b65866fd5 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 11:36:21 +0200 Subject: [PATCH 21/81] Start collecting newStuff --- elm/src/Test/Runner/Node.elm | 2 -- lib/Generate.js | 58 ++---------------------------------- lib/RunTests.js | 9 +++--- lib/Supervisor.js | 35 +++++++++++++++++++++- 4 files changed, 41 insertions(+), 63 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 21a2e887..702d21e3 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -350,8 +350,6 @@ sendResults isFinished testReporter results = |> List.foldl addToKeyValues [] |> Encode.object ) - - -- TODO: Actually care about this in Supervisor , ( "newStuff", Encode.list encodeNewStuff results ) ] |> Encode.encode 0 diff --git a/lib/Generate.js b/lib/Generate.js index b1737e9d..f8db0033 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -5,19 +5,6 @@ const ElmJson = require('./ElmJson'); const Hash = require('./Hash'); const Solve = require('./Solve'); -// Stores all we need to know about a previous run to determine -// which tests haven’t changed and therefore can be skipped. -// TODO: Do we store test results? Or do we somehow only store the ones that passed? -const fingerprintsFileName = 'fingerprints.json'; - -/** - * @typedef { { - fuzz: number, - seed: number, - hashes: Record - } } Fingerprints - */ - const before = fs.readFileSync( path.join(__dirname, '..', 'templates', 'before.js'), 'utf8' @@ -29,17 +16,15 @@ const after = fs.readFileSync( ); /** - * @param { number } fuzz - * @param { number } seed * @param { Array<{ moduleName: string, possiblyTests: Array, }> } testModules * @param { string } pipeFilename * @param { string } dest - * @returns { void } + * @returns { Record } */ -function prepareCompiledJsFile(fuzz, seed, testModules, pipeFilename, dest) { +function prepareCompiledJsFile(testModules, pipeFilename, dest) { const content = fs.readFileSync(dest, 'utf8'); const names = testModules.flatMap((mod) => @@ -50,18 +35,6 @@ function prepareCompiledJsFile(fuzz, seed, testModules, pipeFilename, dest) { const hashes = Hash.calculateHashes(names, content); - /** @type { Fingerprints } */ - const fingerprints = { - fuzz, - seed, - hashes, - }; - - fs.writeFileSync( - path.join(path.dirname(dest), fingerprintsFileName), - JSON.stringify(fingerprints) - ); - const finalContent = ` ${before} var Elm = (function(module) { @@ -80,32 +53,8 @@ ${after} path.join(path.dirname(dest), 'package.json'), JSON.stringify({ type: 'commonjs' }) ); -} -/** - * @param { string } dest - * @returns { Fingerprints } - */ -function readOldFingerprints(dest) { - try { - return JSON.parse( - fs.readFileSync( - path.join(path.dirname(dest), fingerprintsFileName), - 'utf-8' - ) - ); - } catch (error) { - if (error.code !== 'ENOENT') { - console.warn( - `Ignoring bad fingerprints file:\n\n${error.message}\n\nPlease report this issue: https://github.com/rtfeldman/node-test-runner/issues/new` - ); - } - } - return { - fuzz: -1, - seed: -1, - hashes: {}, - }; + return hashes; } // For older versions of elm-explorations/test we need to list every single @@ -451,5 +400,4 @@ module.exports = { generateMainModule, getMainModule, prepareCompiledJsFile, - readOldFingerprints, }; diff --git a/lib/RunTests.js b/lib/RunTests.js index ca78b01f..701c64a1 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -246,8 +246,6 @@ function runTests( progressLogger.log('Compiling'); - const oldFingerprints = Generate.readOldFingerprints(dest); - Generate.generateMainModule( fuzz, seed, @@ -267,9 +265,7 @@ function runTests( report ); - Generate.prepareCompiledJsFile( - fuzz, - seed, + const hashes = Generate.prepareCompiledJsFile( testModules, pipeFilename, dest @@ -280,7 +276,10 @@ function runTests( return await Supervisor.run( packageInfo.version, + hashes, pipeFilename, + fuzz, + seed, report, processes, dest, diff --git a/lib/Supervisor.js b/lib/Supervisor.js index e67bff62..fbeaed16 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -7,14 +7,27 @@ const Report = require('./Report'); /** * @param { string } elmTestVersion + * @param { Record } hashes * @param { string } pipeFilename + * @param { number } fuzz + * @param { number } seed * @param { import('./Report').Report } report * @param { number } processes * @param { string } dest * @param { boolean } watch * @returns { Promise } */ -function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { +function run( + elmTestVersion, + hashes, + pipeFilename, + fuzz, + seed, + report, + processes, + dest, + watch +) { return new Promise(function (resolve) { /** @type { number | null } */ var nextResultToPrint = null; @@ -29,6 +42,18 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { var startingTime = Date.now(); /** @type { Array } */ var workers = []; + /** @type { { fuzzRuns: number, initialSeed: number, fingerprints: Record }> } } */ + var toBePreviousRun = { + fuzzRuns: fuzz, + initialSeed: seed, + fingerprints: {}, + }; + for (var key in hashes) { + toBePreviousRun.fingerprints[key] = { + hash: hashes[key], + outcomes: [], + }; + } /** * @param { any } result This `any` became explicit instead of implicit when migrating from Flow to TypeScript. @@ -152,6 +177,12 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { } }); + for (const newStuff of response.newStuff) { + toBePreviousRun.fingerprints[newStuff.jsDefinitionName].outcomes.push( + newStuff.dictTupleElmCode + ); + } + flushResults(); } @@ -243,6 +274,8 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { .end() ); } + + // TODO: Write `toBePreviousRun`. } // Close all the workers. From 72a015b6eb9492f3abc60f890b369fbd9fd1e698 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 13:12:09 +0200 Subject: [PATCH 22/81] Write PreviousRun module --- elm/src/Test/Runner/Node.elm | 4 +- lib/Generate.js | 94 +++++++++++++++++++++++++++++++----- lib/RunTests.js | 11 ++++- lib/Supervisor.js | 10 +++- 4 files changed, 101 insertions(+), 18 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 702d21e3..7f6b0ccc 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,4 @@ -port module Test.Runner.Node exposing (check, run, TestProgram) +port module Test.Runner.Node exposing (check, run, TestProgram, PreviousRun) {-| @@ -8,7 +8,7 @@ port module Test.Runner.Node exposing (check, run, TestProgram) Runs a test and outputs its results to the console. Exit code is 0 if tests passed and 2 if any failed. Returns 1 if something went wrong. -@docs check, run, TestProgram +@docs check, run, TestProgram, PreviousRun -} diff --git a/lib/Generate.js b/lib/Generate.js index f8db0033..241e31b5 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -191,15 +191,20 @@ function generateElmJson(dependencyProvider, project) { } } +const mainModuleName = ['Test', 'Generated', 'Main']; +const previousRunModuleName = ['Test', 'Generated', 'PreviousRun']; + /** - * @param { string } generatedCodeDir - * @returns { { + * @typedef { { moduleName: string, path: string, - } } + } } Module + * + * @param { string } generatedCodeDir + * @param { Array } moduleName + * @returns { Module } */ -function getMainModule(generatedCodeDir) { - const moduleName = ['Test', 'Generated', 'Main']; +function getModule(generatedCodeDir, moduleName) { return { moduleName: moduleName.join('.'), path: @@ -220,7 +225,7 @@ function getMainModule(generatedCodeDir) { moduleName: string, possiblyTests: Array, }> } testModules - * @param { { moduleName: string, path: string } } mainModule + * @param { Module } mainModule * @param { number } processes * @returns { void } */ @@ -267,6 +272,7 @@ import Test.Reporter.Reporter exposing (Report(..)) import Console.Text exposing (UseColor(..)) import Test.Runner.Node import Test +import ${previousRunModuleName.join('.')} main : Test.Runner.Node.TestProgram main = @@ -351,16 +357,11 @@ function makeOptsCode( , report = ${generateElmReportVariant(report)} , seed = ${seed} , processes = ${processes} +, previousRun = ${previousRunModuleName.join('.')}.previousRun , globs = ${indentAllButFirstLine(' ', makeList(testFileGlobs.map(makeElmString)))} , paths = ${indentAllButFirstLine(' ', makeList(testFilePaths.map(makeElmString)))} --- TODO: Pass actual values here -, previousRun = - { fuzzRuns = 0 - , initialSeed = 0 - , fingerprints = Dict.empty - } } `.trim(); } @@ -395,9 +396,76 @@ function makeElmString(string) { .replace(/\r/g, '\\r')}"`; } +/** + * @param { Module } previousRunModule + * @returns { void } + */ +function ensurePreviousRunModule(previousRunModule) { + if (fs.existsSync(previousRunModule.path)) { + return; + } + generatePreviousRunModule(previousRunModule, { + fuzzRuns: -1, + initialSeed: -1, + fingerprints: {}, + }); +} + +/** + * @typedef { { + fuzzRuns: number, + initialSeed: number, + fingerprints: Record }> + } } PreviousRun + * + * @param { Module } previousRunModule + * @param { PreviousRun } previousRun + * @returns { void } + */ +function generatePreviousRunModule(previousRunModule, previousRun) { + const fingerprintsList = makeList( + Object.entries(previousRun.fingerprints).map( + ([jsIdentifierName, { hash, outcomes }]) => + ` +( ${makeElmString(jsIdentifierName)} +, { hash = ${makeElmString(hash)} + , outcomes = + Dict.fromList + ${indentAllButFirstLine(' ', makeList(outcomes))} + } +) + `.trim() + ) + ); + + const fileContents = ` +module ${previousRunModule.moduleName} exposing (previousRun) + +import Dict +import Test.Runner.Node + +previousRun : Test.Runner.Node.PreviousRun +previousRun = + { fuzzRuns = ${previousRun.fuzzRuns} + , initialSeed = ${previousRun.initialSeed} + , fingerprints = + Dict.fromList + ${indentAllButFirstLine(' ', fingerprintsList)} + } + `.trim(); + + fs.mkdirSync(path.dirname(previousRunModule.path), { recursive: true }); + + fs.writeFileSync(previousRunModule.path, fileContents); +} + module.exports = { + ensurePreviousRunModule, generateElmJson, generateMainModule, - getMainModule, + generatePreviousRunModule, + getModule, + mainModuleName, prepareCompiledJsFile, + previousRunModuleName, }; diff --git a/lib/RunTests.js b/lib/RunTests.js index 701c64a1..263347dc 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -239,7 +239,14 @@ function runTests( runsExecuted++; const pipeFilename = getPipeFilename(runsExecuted); const testModules = await FindTests.findTests(testFilePaths, project); - const mainModule = Generate.getMainModule(project.generatedCodeDir); + const mainModule = Generate.getModule( + project.generatedCodeDir, + Generate.mainModuleName + ); + const previousRunModule = Generate.getModule( + project.generatedCodeDir, + Generate.previousRunModuleName + ); const dest = path.join(project.generatedCodeDir, 'elmTestOutput.js'); Generate.generateElmJson(dependencyProvider, project); @@ -256,6 +263,7 @@ function runTests( mainModule, processes ); + Generate.ensurePreviousRunModule(previousRunModule); await Compile.compile( project.generatedCodeDir, @@ -277,6 +285,7 @@ function runTests( return await Supervisor.run( packageInfo.version, hashes, + previousRunModule, pipeFilename, fuzz, seed, diff --git a/lib/Supervisor.js b/lib/Supervisor.js index fbeaed16..a9c58b4b 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -3,11 +3,13 @@ const child_process = require('child_process'); const fs = require('fs'); const net = require('net'); const split = require('split'); +const Generate = require('./Generate'); const Report = require('./Report'); /** * @param { string } elmTestVersion * @param { Record } hashes + * @param { import('./Generate').Module } previousRunModule * @param { string } pipeFilename * @param { number } fuzz * @param { number } seed @@ -20,6 +22,7 @@ const Report = require('./Report'); function run( elmTestVersion, hashes, + previousRunModule, pipeFilename, fuzz, seed, @@ -42,7 +45,7 @@ function run( var startingTime = Date.now(); /** @type { Array } */ var workers = []; - /** @type { { fuzzRuns: number, initialSeed: number, fingerprints: Record }> } } */ + /** @type { import('./Generate').PreviousRun } */ var toBePreviousRun = { fuzzRuns: fuzz, initialSeed: seed, @@ -275,7 +278,10 @@ function run( ); } - // TODO: Write `toBePreviousRun`. + Generate.generatePreviousRunModule( + previousRunModule, + toBePreviousRun + ); } // Close all the workers. From 5f135522a62bef825aa55f78cef6e30a17f647c9 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 13:30:19 +0200 Subject: [PATCH 23/81] Detect fuzz test --- elm/src/Test/Runner/Node.elm | 34 +++++++++++++++++++++++----------- lib/Generate.js | 18 ++++++++++++++++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 7f6b0ccc..79bf06bf 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -171,7 +171,13 @@ dispatch model startTime = outcome_ Nothing -> - runTestAndCheckIfFuzzTest config.run + let + ( expectations, isFuzzTest ) = + detectFuzzTest config.run + in + { outcome = outcomeFromExpectations expectations + , isFuzzTest = isFuzzTest + } in Time.now |> Task.perform (Complete metadata config.labels outcome startTime) @@ -190,15 +196,21 @@ lastTwoReversed list = Nothing -runTestAndCheckIfFuzzTest : (() -> List Expectation) -> Outcome2 -runTestAndCheckIfFuzzTest run_ = - -- Replace with kernel code that: - -- Resets global `isFuzzTest` var to `false` - -- Reads it again instead of `False` below - -- + patch `fuzzLoop` to set `isFuzzTest` to `true` - { outcome = outcomeFromExpectations (run_ ()) - , isFuzzTest = False - } +{-| The implementation of this function will be replaced in the generated JS +with a version that returns calls the passed function, and detects if it was +a fuzz test. + +If you rename or change this function you also need to update the regex that looks for it. + +-} +detectFuzzTest : (() -> a) -> ( a, Bool ) +detectFuzzTest = + detectFuzzTestHelperReplaceMe___ + + +detectFuzzTestHelperReplaceMe___ : (() -> a) -> ( a, Bool ) +detectFuzzTestHelperReplaceMe___ _ = + Debug.todo "The regex for replacing this Debug.todo in detectFuzzTestHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" update : Msg -> Model -> ( Model, Cmd Msg ) @@ -489,7 +501,7 @@ check = checkHelperReplaceMe___ : a -> String -> String -> b checkHelperReplaceMe___ _ _ _ = - Debug.todo "The regex for replacing this Debug.todo with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" + Debug.todo "The regex for replacing this Debug.todo in checkHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" {-| Run the tests. diff --git a/lib/Generate.js b/lib/Generate.js index 241e31b5..d6faa483 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -69,6 +69,9 @@ const testVariantDefinition = const checkDefinition = /^(var\s+\$author\$project\$Test\$Runner\$Node\$check)\s*=\s*\$author\$project\$Test\$Runner\$Node\$checkHelperReplaceMe___;?$/m; +const detectFuzzTestDefinition = + /^(var\s+\$author\$project\$Test\$Runner\$Node\$detectFuzzTest)\s*=\s*\$author\$project\$Test\$Runner\$Node\$detectFuzzTestHelperReplaceMe___;?$/m; + // For the identifier, this uses the same regex as `REFERENCES_REGEX` in `Hash.js`. const hashPlaceholder = /__elm_test_hash__:([$_][$\w\u0080-\uffff]+)/g; @@ -116,6 +119,21 @@ $1 = F3((test, jsDefinitionName, hash) => ? $elm$core$Maybe$Just({ test, jsDefinitionName, hash, label: _getElmTestLabel(test) }) : $elm$core$Maybe$Nothing ); +`.trim() + ) + .replace( + detectFuzzTestDefinition, + ` +var _elmTestIsFuzzTest = false; +var $elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal = $elm_explorations$test$Test$Fuzz$fuzzLoop; +$elm_explorations$test$Test$Fuzz$fuzzLoop = F2(function (c, state) { + _elmTestIsFuzzTest = true; + return A2($elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal, c, state); +}); +$1 = (f) => { + _elmTestIsFuzzTest = false; + return _Utils_Tuple2(f(null), _elmTestIsFuzzTest); +} `.trim() ) .replace(hashPlaceholder, (_match, name) => hashes[name]) From 7fa5f5a1cc6e4aeab128991c1aad2e04897717d8 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 13:44:35 +0200 Subject: [PATCH 24/81] Add missing imports --- lib/Generate.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/Generate.js b/lib/Generate.js index d6faa483..375b8a00 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -460,6 +460,9 @@ function generatePreviousRunModule(previousRunModule, previousRun) { module ${previousRunModule.moduleName} exposing (previousRun) import Dict +import Test.Distribution exposing (DistributionReport(..)) +import Test.Reporter.TestResults exposing (Outcome(..)) +import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) import Test.Runner.Node previousRun : Test.Runner.Node.PreviousRun From b402578486345efdf2aa7011dd3f17500b254f74 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 13:47:03 +0200 Subject: [PATCH 25/81] Filter out non-tests --- lib/Generate.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/Generate.js b/lib/Generate.js index 375b8a00..50521e7e 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -442,8 +442,11 @@ function ensurePreviousRunModule(previousRunModule) { */ function generatePreviousRunModule(previousRunModule, previousRun) { const fingerprintsList = makeList( - Object.entries(previousRun.fingerprints).map( - ([jsIdentifierName, { hash, outcomes }]) => + Object.entries(previousRun.fingerprints) + // If there are no outcomes, the value wasn’t a test. + // Remember that we collect _all_ exposed values and test them at runtime if they are tests or not. + .filter(([, { outcomes }]) => outcomes.length > 0) + .map(([jsIdentifierName, { hash, outcomes }]) => ` ( ${makeElmString(jsIdentifierName)} , { hash = ${makeElmString(hash)} @@ -453,7 +456,7 @@ function generatePreviousRunModule(previousRunModule, previousRun) { } ) `.trim() - ) + ) ); const fileContents = ` From 6bbf583397e8ddbd9b4e9ed8c69125d96a0f3a15 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 16:31:11 +0200 Subject: [PATCH 26/81] Fix issue where the same function is present in two different recursion chains --- lib/Hash.js | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/lib/Hash.js b/lib/Hash.js index 49d8e68e..3a93e92a 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -18,7 +18,7 @@ const crypto = require('crypto'); function calculateHashes(names, code) { const chunks = parseStep(code); const graph = referencesStep(names, chunks); - return hashStep(names, graph); + return hashStep(names, chunks, graph); } /** @@ -88,7 +88,7 @@ function parseStep(code) { * Merges recursive chains into single items, so that the output is an acyclic graph. * * @typedef { { - code: Array, + definitions: Set, references: Set, } } Node * @@ -119,7 +119,7 @@ function referencesStep(names, chunks) { } /** @type { Node } */ const node = { - code: [chunk], + definitions: new Set([name]), references: new Set( references.filter( (reference) => @@ -148,21 +148,28 @@ function referencesStep(names, chunks) { // containing the code and references of all the involved functions. // This is how we make the graph acyclic. const chain = seen.slice(index); - /** @type { Node } */ - const newNode = { - code: [], - references: new Set(), - }; + /** @type { Set } */ + const definitions = new Set(); for (const chainName of chain) { // Note: Since `chainName` is a name we have already visited, // we know that we have inserted a node for it. const chainNode = graph[chainName]; - for (const code of chainNode.code) { - newNode.code.push(code); + for (const chainName_ of chainNode.definitions) { + definitions.add(chainName_); } + } + /** @type { Node } */ + const newNode = { + definitions, + references: new Set(), + }; + for (const chainName of definitions) { + // Note: `chainName` comes from `.definitions` on nodes, + // and those only referer to things already inserted in `graph`. + const chainNode = graph[chainName]; for (const chainReference of chainNode.references) { // Skip direct recursion. - if (!chain.includes(chainReference)) { + if (!definitions.has(chainReference)) { newNode.references.add(chainReference); } } @@ -185,10 +192,11 @@ function referencesStep(names, chunks) { /** * @param { Array } names + * @param { Record } chunks * @param { Record } graph * @returns { Record } */ -function hashStep(names, graph) { +function hashStep(names, chunks, graph) { /** @type { Record } */ const hashes = {}; @@ -214,8 +222,9 @@ function hashStep(names, graph) { // amount of time used by `getOrCalculateHash`. `sha256` is one of // the ones being about 10 ms faster than the slowest ones. const hashObject = crypto.createHash('sha256'); - for (const code of Array.from(node.code).sort()) { - hashObject.update(code); + for (const name of Array.from(node.definitions).sort()) { + // Note: Nodes in `graph` only refer to things that exist in `chunks`. + hashObject.update(chunks[name]); } for (const reference of Array.from(node.references).sort()) { hashObject.update(getOrCalculateHash(reference)); From 5ffe8412e6f1181d74b60416505c8708191253da Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 16:59:30 +0200 Subject: [PATCH 27/81] Temp time measurements --- elm/src/Test/Runner/Node.elm | 4 +++- lib/Generate.js | 4 ++++ lib/RunTests.js | 33 ++++++++++++++++++++++++++++----- lib/Supervisor.js | 8 ++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 79bf06bf..088ccea8 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -286,7 +286,9 @@ update msg ({ testReporter } as model) = Complete metadata labels outcome2 startTime endTime -> let duration = - Time.posixToMillis endTime - Time.posixToMillis startTime + Time.posixToMillis endTime + - Time.posixToMillis startTime + |> Debug.log "duration" results = ( model.nextTestToRun diff --git a/lib/Generate.js b/lib/Generate.js index 50521e7e..619b284b 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -25,7 +25,9 @@ const after = fs.readFileSync( * @returns { Record } */ function prepareCompiledJsFile(testModules, pipeFilename, dest) { + console.time('fs.readFileSync'); const content = fs.readFileSync(dest, 'utf8'); + console.timeEnd('fs.readFileSync'); const names = testModules.flatMap((mod) => mod.possiblyTests.map((test) => @@ -33,7 +35,9 @@ function prepareCompiledJsFile(testModules, pipeFilename, dest) { ) ); + console.time('Hash.calculateHashes'); const hashes = Hash.calculateHashes(names, content); + console.timeEnd('Hash.calculateHashes'); const finalContent = ` ${before} diff --git a/lib/RunTests.js b/lib/RunTests.js index 263347dc..9096f088 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -57,26 +57,26 @@ function makeProgressLogger(report, clearConsole) { log(message) { items.push(message); if (!Report.isMachineReadable(report)) { - process.stdout.write(`${items.join(' > ')}\r`); + // process.stdout.write(`${items.join(' > ')}\r`); } }, newLine() { items.length = 0; if (!Report.isMachineReadable(report)) { - process.stdout.write('\n'); + // process.stdout.write('\n'); } }, overwrite(message) { items.length = 0; items.push(message); if (!Report.isMachineReadable(report)) { - process.stdout.write(`${message}\r`); + // process.stdout.write(`${message}\r`); } }, clearLine() { items.length = 0; if (!Report.isMachineReadable(report)) { - readline.clearLine(process.stdout, 0); + // readline.clearLine(process.stdout, 0); } }, clearConsole() { @@ -214,10 +214,17 @@ function runTests( // Files may be changed, added or removed so always re-create project info // from disk to stay fresh. + console.time('Project.init'); const project = Project.init(projectRootDir, packageInfo.version); + console.timeEnd('Project.init'); + console.time('ElmJson.requireElmTestPackage'); ElmJson.requireElmTestPackage(projectRootDir, project.elmJson); + console.timeEnd('ElmJson.requireElmTestPackage'); + console.time('Project.validateTestsSourceDirs'); Project.validateTestsSourceDirs(project); + console.timeEnd('Project.validateTestsSourceDirs'); + console.time('FindTests.resolveGlobs'); const testFilePaths = FindTests.resolveGlobs( testFileGlobs.length === 0 ? [project.testsDir] : testFileGlobs, project.rootDir @@ -228,6 +235,7 @@ function runTests( FindTests.noFilesFoundError(project.rootDir, testFileGlobs) ); } + console.timeEnd('FindTests.resolveGlobs'); if (watcher !== undefined) { const diff = diffArrays(watchedPaths, project.testsSourceDirs); @@ -238,7 +246,9 @@ function runTests( runsExecuted++; const pipeFilename = getPipeFilename(runsExecuted); + console.time('FindTests.findTests'); const testModules = await FindTests.findTests(testFilePaths, project); + console.timeEnd('FindTests.findTests'); const mainModule = Generate.getModule( project.generatedCodeDir, Generate.mainModuleName @@ -249,10 +259,13 @@ function runTests( ); const dest = path.join(project.generatedCodeDir, 'elmTestOutput.js'); + console.time('Generate.generateElmJson'); Generate.generateElmJson(dependencyProvider, project); + console.timeEnd('Generate.generateElmJson'); progressLogger.log('Compiling'); + console.time('Generate.generateMainModule'); Generate.generateMainModule( fuzz, seed, @@ -263,8 +276,12 @@ function runTests( mainModule, processes ); + console.timeEnd('Generate.generateMainModule'); + console.time('Generate.ensurePreviousRunModule'); Generate.ensurePreviousRunModule(previousRunModule); + console.timeEnd('Generate.ensurePreviousRunModule'); + console.time('Compile.compile'); await Compile.compile( project.generatedCodeDir, mainModule.path, @@ -272,17 +289,21 @@ function runTests( pathToElmBinary, report ); + console.timeEnd('Compile.compile'); + console.time('Generate.prepareCompiledJsFile'); const hashes = Generate.prepareCompiledJsFile( testModules, pipeFilename, dest ); + console.timeEnd('Generate.prepareCompiledJsFile'); progressLogger.log('Starting tests'); progressLogger.newLine(); - return await Supervisor.run( + console.time('Supervisor.run'); + const r = await Supervisor.run( packageInfo.version, hashes, previousRunModule, @@ -294,6 +315,8 @@ function runTests( dest, watch ); + console.timeEnd('Supervisor.run'); + return r; } catch (err) { progressLogger.newLine(); console.error(err.message); diff --git a/lib/Supervisor.js b/lib/Supervisor.js index a9c58b4b..a8dde158 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -57,6 +57,8 @@ function run( outcomes: [], }; } + var sentMessages = 0; + var receivedMessages = 0; /** * @param { any } result This `any` became explicit instead of implicit when migrating from Flow to TypeScript. @@ -209,6 +211,7 @@ function run( } var response = JSON.parse(data); + receivedMessages++; switch (response.type) { case 'FINISHED': @@ -219,6 +222,7 @@ function run( // If all the workers have finished, print the summmary. if (finishedWorkers === workers.length) { + sentMessages++; socket.write( JSON.stringify({ type: 'SUMMARY', @@ -282,6 +286,9 @@ function run( previousRunModule, toBePreviousRun ); + + console.log('sentMessages', sentMessages); + console.log('receivedMessages', receivedMessages); } // Close all the workers. @@ -321,6 +328,7 @@ function run( } }); + sentMessages++; socket.write(JSON.stringify({ type: 'TEST', index: initializedWorkers })); initializedWorkers++; From 073ede957c19c919488e474b87a19c35f4180a2d Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 17:00:17 +0200 Subject: [PATCH 28/81] TODOs --- lib/Generate.js | 1 + lib/Hash.js | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/Generate.js b/lib/Generate.js index 619b284b..58d0b50d 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -128,6 +128,7 @@ $1 = F3((test, jsDefinitionName, hash) => .replace( detectFuzzTestDefinition, ` +// TODO: isFuzzTest is never true var _elmTestIsFuzzTest = false; var $elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal = $elm_explorations$test$Test$Fuzz$fuzzLoop; $elm_explorations$test$Test$Fuzz$fuzzLoop = F2(function (c, state) { diff --git a/lib/Hash.js b/lib/Hash.js index 3a93e92a..582efb81 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -16,6 +16,7 @@ const crypto = require('crypto'); * @returns { Record } */ function calculateHashes(names, code) { + // TODO: Too much recursion on elm-review-simplify const chunks = parseStep(code); const graph = referencesStep(names, chunks); return hashStep(names, chunks, graph); From 68e3dbc0d16af6cb031791f7fb0a85a18d9e98f2 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 21:58:31 +0200 Subject: [PATCH 29/81] Handle error when no fuzz tests --- lib/Generate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Generate.js b/lib/Generate.js index 58d0b50d..d781eab2 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -130,8 +130,8 @@ $1 = F3((test, jsDefinitionName, hash) => ` // TODO: isFuzzTest is never true var _elmTestIsFuzzTest = false; -var $elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal = $elm_explorations$test$Test$Fuzz$fuzzLoop; -$elm_explorations$test$Test$Fuzz$fuzzLoop = F2(function (c, state) { +var $elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal = typeof $elm_explorations$test$Test$Fuzz$fuzzLoop === "function" ? $elm_explorations$test$Test$Fuzz$fuzzLoop : undefined; +var $elm_explorations$test$Test$Fuzz$fuzzLoop = F2(function (c, state) { _elmTestIsFuzzTest = true; return A2($elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal, c, state); }); From 374ff905e7734611089eded9fd551795bb5c50e0 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Thu, 16 Jul 2026 22:08:02 +0200 Subject: [PATCH 30/81] Fix isFuzzTest never being True --- lib/Generate.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/Generate.js b/lib/Generate.js index d781eab2..21b42411 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -76,6 +76,9 @@ const checkDefinition = const detectFuzzTestDefinition = /^(var\s+\$author\$project\$Test\$Runner\$Node\$detectFuzzTest)\s*=\s*\$author\$project\$Test\$Runner\$Node\$detectFuzzTestHelperReplaceMe___;?$/m; +const fuzzLoopDefinition = + /^var \$elm_explorations\$test\$Test\$Fuzz\$fuzzLoop\s*=\s*F\d\(\s*function\s*\([^()]*\)\s*\{/m; + // For the identifier, this uses the same regex as `REFERENCES_REGEX` in `Hash.js`. const hashPlaceholder = /__elm_test_hash__:([$_][$\w\u0080-\uffff]+)/g; @@ -128,19 +131,14 @@ $1 = F3((test, jsDefinitionName, hash) => .replace( detectFuzzTestDefinition, ` -// TODO: isFuzzTest is never true var _elmTestIsFuzzTest = false; -var $elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal = typeof $elm_explorations$test$Test$Fuzz$fuzzLoop === "function" ? $elm_explorations$test$Test$Fuzz$fuzzLoop : undefined; -var $elm_explorations$test$Test$Fuzz$fuzzLoop = F2(function (c, state) { - _elmTestIsFuzzTest = true; - return A2($elm_explorations$test$Test$Fuzz$fuzzLoop_elmTestOriginal, c, state); -}); $1 = (f) => { _elmTestIsFuzzTest = false; return _Utils_Tuple2(f(null), _elmTestIsFuzzTest); } `.trim() ) + .replace(fuzzLoopDefinition, '$& _elmTestIsFuzzTest = true;') .replace(hashPlaceholder, (_match, name) => hashes[name]) ); } From 9e6adcaf829368bc2e4a2ea1866ce9e27268bc6f Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Fri, 17 Jul 2026 10:31:51 +0200 Subject: [PATCH 31/81] Fix recursion with Tarjan --- lib/Hash.js | 145 ++++++++++++++++++++------------------------------ lib/Tarjan.js | 83 +++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 86 deletions(-) create mode 100644 lib/Tarjan.js diff --git a/lib/Hash.js b/lib/Hash.js index 582efb81..63d1aa24 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -1,4 +1,5 @@ const crypto = require('crypto'); +const Tarjan = require('./Tarjan'); /** * Pass in an array of definition names, such as `["$author$project$MyTest$suite"]`, @@ -16,9 +17,9 @@ const crypto = require('crypto'); * @returns { Record } */ function calculateHashes(names, code) { - // TODO: Too much recursion on elm-review-simplify const chunks = parseStep(code); - const graph = referencesStep(names, chunks); + const graph = graphStep(chunks); + makeAcyclicStep(graph); return hashStep(names, chunks, graph); } @@ -85,110 +86,82 @@ function parseStep(code) { } /** - * Resolves references for everything reachable from `names` in `chunks`. - * Merges recursive chains into single items, so that the output is an acyclic graph. + * Parses references in all `chunks`. * * @typedef { { definitions: Set, references: Set, } } Node * - * @param { Array } names * @param { Record } chunks * @returns { Record } */ -function referencesStep(names, chunks) { +function graphStep(chunks) { /** @type { Record } */ const graph = {}; - /** - * @param { string } name - * @param { string } chunk - * @param { Array } seenPreviously - * @returns { void } - */ - const createNode = (name, chunk, seenPreviously) => { - // Already processed. - if (name in graph) { - return; - } - - // Create a node in the graph. - const references = chunk.match(REFERENCES_REGEX); - if (references === null) { - throw new Error(`No references found in chunk for ${name}:\n${chunk}`); - } - /** @type { Node } */ - const node = { + for (const name in chunks) { + const chunk = chunks[name]; + const tokens = chunk.match(REFERENCES_REGEX); + graph[name] = { definitions: new Set([name]), - references: new Set( - references.filter( - (reference) => - // Skip string literals and comments and take only identifiers – see `REFERENCES_REGEX`. - (reference.startsWith('$') || reference.startsWith('_')) && - // Skip direct recursion. - reference !== name && - // Only care about references to stuff defined in `chunks`. - reference in chunks - ) - ), + references: + // Not all chunks contains any tokens that we care about (such as the `F` helper). + tokens === null + ? new Set() + : new Set( + tokens.filter( + (token) => + // Skip string literals and comments and take only identifiers – see `REFERENCES_REGEX`. + (token.startsWith('$') || token.startsWith('_')) && + // Skip direct recursion. + token !== name && + // Only care about references to stuff defined in `chunks`. + token in chunks + ) + ), }; - graph[name] = node; - - // Create nodes in the graph for all references. - const seen = [...seenPreviously, name]; - for (const reference of node.references) { - // Note: We already checked `reference in chunks` when constructing `node.references`. - const referenceChunk = chunks[reference]; - let index = seen.indexOf(reference); - if (index === -1) { - createNode(reference, referenceChunk, seen); - } else { - // A chain of indirect recursion was found! - // Replace all the involved functions with the same node, - // containing the code and references of all the involved functions. - // This is how we make the graph acyclic. - const chain = seen.slice(index); - /** @type { Set } */ - const definitions = new Set(); - for (const chainName of chain) { - // Note: Since `chainName` is a name we have already visited, - // we know that we have inserted a node for it. - const chainNode = graph[chainName]; - for (const chainName_ of chainNode.definitions) { - definitions.add(chainName_); + } + + return graph; +} + +/** + * Merges recursive chains into single items, so that the output is an acyclic graph. + * + * @param { Record } graph + * @returns { void } + */ +function makeAcyclicStep(graph) { + const scc = Tarjan.stronglyConnectedComponents({ + keys: () => Object.keys(graph), + get: (key) => graph[key].references, + }); + + for (const chain of scc) { + if (chain.size > 1) { + // A chain of indirect recursion was found! + // Replace all the involved functions with the same node, + // containing the definitions and references of all the involved functions. + // This is how we make the graph acyclic. + /** @type { Set } */ + const references = new Set(); + for (const chainName of chain) { + // Note: `chainName` comes from keys in the graph. + const chainNode = graph[chainName]; + for (const chainReference of chainNode.references) { + // Skip direct recursion. + if (!chain.has(chainReference)) { + references.add(chainReference); } } - /** @type { Node } */ - const newNode = { - definitions, - references: new Set(), + graph[chainName] = { + definitions: chain, + references, }; - for (const chainName of definitions) { - // Note: `chainName` comes from `.definitions` on nodes, - // and those only referer to things already inserted in `graph`. - const chainNode = graph[chainName]; - for (const chainReference of chainNode.references) { - // Skip direct recursion. - if (!definitions.has(chainReference)) { - newNode.references.add(chainReference); - } - } - graph[chainName] = newNode; - } } } - }; - - for (const name of names) { - const chunk = chunks[name]; - if (chunk === undefined) { - throw new Error(`Could not find ${name} in the compiled code!`); - } - createNode(name, chunk, []); } - - return graph; } /** diff --git a/lib/Tarjan.js b/lib/Tarjan.js new file mode 100644 index 00000000..ba19fc4a --- /dev/null +++ b/lib/Tarjan.js @@ -0,0 +1,83 @@ +/** +Based on @rtsao/scc@1.1.0 +https://github.com/rtsao/scc/blob/317512b2b6615736ad9bd3f23e8cee739ff44cf6/index.js + +MIT License + +Copyright (c) 2019 Ryan Tsao + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +/** + * Find strongly connected components (SCC) of a directed graph using Tarjan's algorithm. + * + * Adapted from https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm#The_algorithm_in_pseudocode + * + * @typedef { { + keys: () => Array, + get: (key: string) => Set, + } } Graph + * + * @param { Graph } graph + * @returns { Array> } + */ +function stronglyConnectedComponents(graph) { + const indices = new Map(); + const lowLinks = new Map(); + const onStack = new Set(); + /** @type { Array } */ + const stack = []; + /** @type { Array> } */ + const scc = []; + let idx = 0; + + /** + * @param { string } v + * @returns { void } + */ + function strongConnect(v) { + indices.set(v, idx); + lowLinks.set(v, idx); + idx++; + stack.push(v); + onStack.add(v); + + const deps = graph.get(v); + for (const dep of deps) { + if (!indices.has(dep)) { + strongConnect(dep); + lowLinks.set(v, Math.min(lowLinks.get(v), lowLinks.get(dep))); + } else if (onStack.has(dep)) { + lowLinks.set(v, Math.min(lowLinks.get(v), indices.get(dep))); + } + } + + if (lowLinks.get(v) === indices.get(v)) { + const vertices = new Set(); + let w = null; + while (v !== w) { + w = stack.pop(); + onStack.delete(w); + vertices.add(w); + } + scc.push(vertices); + } + } + + for (const v of graph.keys()) { + if (!indices.has(v)) { + strongConnect(v); + } + } + + return scc; +} + +module.exports = { + stronglyConnectedComponents, +}; From f0edf35c1a3fca638c7ec8dd425c63f4b200dd70 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Fri, 17 Jul 2026 10:33:58 +0200 Subject: [PATCH 32/81] Revert "Temp time measurements" This reverts commit 5ffe8412e6f1181d74b60416505c8708191253da. --- elm/src/Test/Runner/Node.elm | 4 +--- lib/Generate.js | 4 ---- lib/RunTests.js | 33 +++++---------------------------- lib/Supervisor.js | 8 -------- 4 files changed, 6 insertions(+), 43 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 088ccea8..79bf06bf 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -286,9 +286,7 @@ update msg ({ testReporter } as model) = Complete metadata labels outcome2 startTime endTime -> let duration = - Time.posixToMillis endTime - - Time.posixToMillis startTime - |> Debug.log "duration" + Time.posixToMillis endTime - Time.posixToMillis startTime results = ( model.nextTestToRun diff --git a/lib/Generate.js b/lib/Generate.js index 21b42411..7d356820 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -25,9 +25,7 @@ const after = fs.readFileSync( * @returns { Record } */ function prepareCompiledJsFile(testModules, pipeFilename, dest) { - console.time('fs.readFileSync'); const content = fs.readFileSync(dest, 'utf8'); - console.timeEnd('fs.readFileSync'); const names = testModules.flatMap((mod) => mod.possiblyTests.map((test) => @@ -35,9 +33,7 @@ function prepareCompiledJsFile(testModules, pipeFilename, dest) { ) ); - console.time('Hash.calculateHashes'); const hashes = Hash.calculateHashes(names, content); - console.timeEnd('Hash.calculateHashes'); const finalContent = ` ${before} diff --git a/lib/RunTests.js b/lib/RunTests.js index 9096f088..263347dc 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -57,26 +57,26 @@ function makeProgressLogger(report, clearConsole) { log(message) { items.push(message); if (!Report.isMachineReadable(report)) { - // process.stdout.write(`${items.join(' > ')}\r`); + process.stdout.write(`${items.join(' > ')}\r`); } }, newLine() { items.length = 0; if (!Report.isMachineReadable(report)) { - // process.stdout.write('\n'); + process.stdout.write('\n'); } }, overwrite(message) { items.length = 0; items.push(message); if (!Report.isMachineReadable(report)) { - // process.stdout.write(`${message}\r`); + process.stdout.write(`${message}\r`); } }, clearLine() { items.length = 0; if (!Report.isMachineReadable(report)) { - // readline.clearLine(process.stdout, 0); + readline.clearLine(process.stdout, 0); } }, clearConsole() { @@ -214,17 +214,10 @@ function runTests( // Files may be changed, added or removed so always re-create project info // from disk to stay fresh. - console.time('Project.init'); const project = Project.init(projectRootDir, packageInfo.version); - console.timeEnd('Project.init'); - console.time('ElmJson.requireElmTestPackage'); ElmJson.requireElmTestPackage(projectRootDir, project.elmJson); - console.timeEnd('ElmJson.requireElmTestPackage'); - console.time('Project.validateTestsSourceDirs'); Project.validateTestsSourceDirs(project); - console.timeEnd('Project.validateTestsSourceDirs'); - console.time('FindTests.resolveGlobs'); const testFilePaths = FindTests.resolveGlobs( testFileGlobs.length === 0 ? [project.testsDir] : testFileGlobs, project.rootDir @@ -235,7 +228,6 @@ function runTests( FindTests.noFilesFoundError(project.rootDir, testFileGlobs) ); } - console.timeEnd('FindTests.resolveGlobs'); if (watcher !== undefined) { const diff = diffArrays(watchedPaths, project.testsSourceDirs); @@ -246,9 +238,7 @@ function runTests( runsExecuted++; const pipeFilename = getPipeFilename(runsExecuted); - console.time('FindTests.findTests'); const testModules = await FindTests.findTests(testFilePaths, project); - console.timeEnd('FindTests.findTests'); const mainModule = Generate.getModule( project.generatedCodeDir, Generate.mainModuleName @@ -259,13 +249,10 @@ function runTests( ); const dest = path.join(project.generatedCodeDir, 'elmTestOutput.js'); - console.time('Generate.generateElmJson'); Generate.generateElmJson(dependencyProvider, project); - console.timeEnd('Generate.generateElmJson'); progressLogger.log('Compiling'); - console.time('Generate.generateMainModule'); Generate.generateMainModule( fuzz, seed, @@ -276,12 +263,8 @@ function runTests( mainModule, processes ); - console.timeEnd('Generate.generateMainModule'); - console.time('Generate.ensurePreviousRunModule'); Generate.ensurePreviousRunModule(previousRunModule); - console.timeEnd('Generate.ensurePreviousRunModule'); - console.time('Compile.compile'); await Compile.compile( project.generatedCodeDir, mainModule.path, @@ -289,21 +272,17 @@ function runTests( pathToElmBinary, report ); - console.timeEnd('Compile.compile'); - console.time('Generate.prepareCompiledJsFile'); const hashes = Generate.prepareCompiledJsFile( testModules, pipeFilename, dest ); - console.timeEnd('Generate.prepareCompiledJsFile'); progressLogger.log('Starting tests'); progressLogger.newLine(); - console.time('Supervisor.run'); - const r = await Supervisor.run( + return await Supervisor.run( packageInfo.version, hashes, previousRunModule, @@ -315,8 +294,6 @@ function runTests( dest, watch ); - console.timeEnd('Supervisor.run'); - return r; } catch (err) { progressLogger.newLine(); console.error(err.message); diff --git a/lib/Supervisor.js b/lib/Supervisor.js index a8dde158..a9c58b4b 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -57,8 +57,6 @@ function run( outcomes: [], }; } - var sentMessages = 0; - var receivedMessages = 0; /** * @param { any } result This `any` became explicit instead of implicit when migrating from Flow to TypeScript. @@ -211,7 +209,6 @@ function run( } var response = JSON.parse(data); - receivedMessages++; switch (response.type) { case 'FINISHED': @@ -222,7 +219,6 @@ function run( // If all the workers have finished, print the summmary. if (finishedWorkers === workers.length) { - sentMessages++; socket.write( JSON.stringify({ type: 'SUMMARY', @@ -286,9 +282,6 @@ function run( previousRunModule, toBePreviousRun ); - - console.log('sentMessages', sentMessages); - console.log('receivedMessages', receivedMessages); } // Close all the workers. @@ -328,7 +321,6 @@ function run( } }); - sentMessages++; socket.write(JSON.stringify({ type: 'TEST', index: initializedWorkers })); initializedWorkers++; From 27623365eaf651521d5cfe36684f5675f660b998 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Fri, 17 Jul 2026 10:58:51 +0200 Subject: [PATCH 33/81] Detect Debug.log --- elm/src/Test/Reporter/JUnit.elm | 1 + elm/src/Test/Reporter/TestResults.elm | 1 + elm/src/Test/Runner/Node.elm | 38 +++++++++++++++++---------- lib/Generate.js | 14 +++++++--- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/elm/src/Test/Reporter/JUnit.elm b/elm/src/Test/Reporter/JUnit.elm index 07c97f6e..00c2f3ea 100644 --- a/elm/src/Test/Reporter/JUnit.elm +++ b/elm/src/Test/Reporter/JUnit.elm @@ -134,6 +134,7 @@ encodeExtraFailure _ = , outcome = Failed [] , jsDefinitionName = "" , isFuzzTest = False + , usedDebugLog = False } diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index 2f4be226..5b4b0f77 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -25,6 +25,7 @@ type alias TestResult = , duration : Int -- in milliseconds , jsDefinitionName : String , isFuzzTest : Bool + , usedDebugLog : Bool } diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 79bf06bf..88e65a3c 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -117,8 +117,9 @@ type alias Fingerprints = type alias Outcome2 = - { isFuzzTest : Bool - , outcome : Outcome + { outcome : Outcome + , isFuzzTest : Bool + , usedDebugLog : Bool } @@ -150,9 +151,11 @@ dispatch model startTime = |> Maybe.andThen (\outcome_ -> if - not outcome_.isFuzzTest - || ((model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) - && (model.runInfo.initialSeed == model.previousRun.initialSeed) + not outcome_.usedDebugLog + && (not outcome_.isFuzzTest + || ((model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) + && (model.runInfo.initialSeed == model.previousRun.initialSeed) + ) ) then Just outcome_ @@ -172,11 +175,12 @@ dispatch model startTime = Nothing -> let - ( expectations, isFuzzTest ) = - detectFuzzTest config.run + ( expectations, isFuzzTest, usedDebugLog ) = + detectFuzzTestAndDebugLog config.run in { outcome = outcomeFromExpectations expectations , isFuzzTest = isFuzzTest + , usedDebugLog = usedDebugLog } in Time.now @@ -203,14 +207,14 @@ a fuzz test. If you rename or change this function you also need to update the regex that looks for it. -} -detectFuzzTest : (() -> a) -> ( a, Bool ) -detectFuzzTest = - detectFuzzTestHelperReplaceMe___ +detectFuzzTestAndDebugLog : (() -> a) -> ( a, Bool, Bool ) +detectFuzzTestAndDebugLog = + detectFuzzTestAndDebugLogHelperReplaceMe___ -detectFuzzTestHelperReplaceMe___ : (() -> a) -> ( a, Bool ) -detectFuzzTestHelperReplaceMe___ _ = - Debug.todo "The regex for replacing this Debug.todo in detectFuzzTestHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" +detectFuzzTestAndDebugLogHelperReplaceMe___ : (() -> a) -> ( a, Bool, Bool ) +detectFuzzTestAndDebugLogHelperReplaceMe___ _ = + Debug.todo "The regex for replacing this Debug.todo in detectFuzzTestAndDebugLogHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" update : Msg -> Model -> ( Model, Cmd Msg ) @@ -295,6 +299,7 @@ update msg ({ testReporter } as model) = , duration = duration , jsDefinitionName = metadata.jsDefinitionName , isFuzzTest = outcome2.isFuzzTest + , usedDebugLog = outcome2.usedDebugLog } ) :: model.results @@ -348,7 +353,12 @@ sendResults isFinished testReporter results = let dictTuple : ( List String, Outcome2 ) dictTuple = - ( result.labels, { isFuzzTest = result.isFuzzTest, outcome = result.outcome } ) + ( result.labels + , { outcome = result.outcome + , isFuzzTest = result.isFuzzTest + , usedDebugLog = result.usedDebugLog + } + ) in Encode.object [ ( "jsDefinitionName", Encode.string result.jsDefinitionName ) diff --git a/lib/Generate.js b/lib/Generate.js index 7d356820..6d48f1a5 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -69,12 +69,15 @@ const testVariantDefinition = const checkDefinition = /^(var\s+\$author\$project\$Test\$Runner\$Node\$check)\s*=\s*\$author\$project\$Test\$Runner\$Node\$checkHelperReplaceMe___;?$/m; -const detectFuzzTestDefinition = - /^(var\s+\$author\$project\$Test\$Runner\$Node\$detectFuzzTest)\s*=\s*\$author\$project\$Test\$Runner\$Node\$detectFuzzTestHelperReplaceMe___;?$/m; +const detectFuzzTestAndDebugLogDefinition = + /^(var\s+\$author\$project\$Test\$Runner\$Node\$detectFuzzTestAndDebugLog)\s*=\s*\$author\$project\$Test\$Runner\$Node\$detectFuzzTestAndDebugLogHelperReplaceMe___;?$/m; const fuzzLoopDefinition = /^var \$elm_explorations\$test\$Test\$Fuzz\$fuzzLoop\s*=\s*F\d\(\s*function\s*\([^()]*\)\s*\{/m; +const debugLogDefinition = + /^var _Debug_log\s*=\s*F\d\(\s*function\s*\([^()]*\)\s*\{/m; + // For the identifier, this uses the same regex as `REFERENCES_REGEX` in `Hash.js`. const hashPlaceholder = /__elm_test_hash__:([$_][$\w\u0080-\uffff]+)/g; @@ -125,16 +128,19 @@ $1 = F3((test, jsDefinitionName, hash) => `.trim() ) .replace( - detectFuzzTestDefinition, + detectFuzzTestAndDebugLogDefinition, ` var _elmTestIsFuzzTest = false; +var _elmTestUsedDebugLog = false; $1 = (f) => { _elmTestIsFuzzTest = false; - return _Utils_Tuple2(f(null), _elmTestIsFuzzTest); + _elmTestUsedDebugLog = false; + return _Utils_Tuple3(f(null), _elmTestIsFuzzTest, _elmTestUsedDebugLog); } `.trim() ) .replace(fuzzLoopDefinition, '$& _elmTestIsFuzzTest = true;') + .replace(debugLogDefinition, '$& _elmTestUsedDebugLog = true;') .replace(hashPlaceholder, (_match, name) => hashes[name]) ); } From 9be3d54ed0ca089d8c7fe11ba6854a3e647105fe Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Fri, 17 Jul 2026 14:42:38 +0200 Subject: [PATCH 34/81] Support Test.concat --- elm/src/Test/Runner/Node.elm | 19 ++++++++++++------- lib/Generate.js | 13 ++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 88e65a3c..3511eeb6 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -18,6 +18,7 @@ import Json.Decode as Decode import Json.Encode as Encode import Platform import Random +import Set exposing (Set) import Task import Test exposing (Test) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) @@ -137,8 +138,7 @@ dispatch model startTime = Just metadata_ -> metadata_ - -- This should not happen: All tests should have metadata. - -- TODO: Can we get here for `Test.todo`? + -- TODO: We can get here for a bare `Test.todo`. Needs to be handled somehow. Nothing -> { jsDefinitionName = "MISSING:" ++ Debug.toString config.labels, hash = "" } @@ -494,7 +494,7 @@ type alias TestWithMetadata = { test : Test , jsDefinitionName : String , hash : String - , label : String + , labels : Set String } @@ -537,11 +537,16 @@ run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = Just ( Test.describe moduleName moduleTests , moduleTestsWithMetadata - |> List.map + |> List.concatMap (\data -> - ( ( moduleName, data.label ) - , { jsDefinitionName = data.jsDefinitionName, hash = data.hash } - ) + data.labels + |> Set.toList + |> List.map + (\label -> + ( ( moduleName, label ) + , { jsDefinitionName = data.jsDefinitionName, hash = data.hash } + ) + ) ) ) ) diff --git a/lib/Generate.js b/lib/Generate.js index 6d48f1a5..c7ed35dc 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -113,16 +113,15 @@ function addKernelTestChecking(hashes, content) { .replace(testVariantDefinition, '$&__elmTestSymbol: __elmTestSymbol, ') .replace( checkDefinition, + // Abuse `$elm_explorations$test$Test$Internal$duplicatedName` to get the test name(s). + // Usually, a `Test` has a single name (from `test`, `fuzz` or `describe`), but `Test.concat` + // does not have a name – instead we need to use the names of all the tests it concatenates + // (recursively). `$elm_explorations$test$Test$Internal$duplicatedName` returns the unique + // names when there are no duplicates – and there shouldn’t be, because that is not constructable. ` -function _getElmTestLabel(test) { - while (typeof test.a !== "string" && typeof test.a !== undefined) { - test = test.a; - } - return test.a || ""; -} $1 = F3((test, jsDefinitionName, hash) => test && test.__elmTestSymbol === __elmTestSymbol - ? $elm$core$Maybe$Just({ test, jsDefinitionName, hash, label: _getElmTestLabel(test) }) + ? $elm$core$Maybe$Just({ test, jsDefinitionName, hash, labels: $elm_explorations$test$Test$Internal$duplicatedName(_List_Cons(test, _List_Nil)).a }) : $elm$core$Maybe$Nothing ); `.trim() From e4b118ff3bf0b51cfbd72950377a5d232961da47 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Fri, 17 Jul 2026 14:46:42 +0200 Subject: [PATCH 35/81] Support bare Test.todo --- elm/src/Test/Runner/Node.elm | 6 ++++-- lib/Supervisor.js | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 3511eeb6..1288cafd 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -138,9 +138,11 @@ dispatch model startTime = Just metadata_ -> metadata_ - -- TODO: We can get here for a bare `Test.todo`. Needs to be handled somehow. + -- A `Test.todo` not nested under any `Test.describe` ends up here (`config.labels` only + -- contains _one_ label – the one added automatically to each module). + -- Luckily, there is not really much speed gain to caching `Test.todo`s. Nothing -> - { jsDefinitionName = "MISSING:" ++ Debug.toString config.labels, hash = "" } + { jsDefinitionName = "", hash = "" } maybeCachedOutcome = Dict.get metadata.jsDefinitionName model.previousRun.fingerprints diff --git a/lib/Supervisor.js b/lib/Supervisor.js index a9c58b4b..8464beba 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -181,9 +181,12 @@ function run( }); for (const newStuff of response.newStuff) { - toBePreviousRun.fingerprints[newStuff.jsDefinitionName].outcomes.push( - newStuff.dictTupleElmCode - ); + // Ignore bare `Test.todo`, which use the empty string – see Node.elm. + if (newStuff.jsDefinitionName !== '') { + toBePreviousRun.fingerprints[newStuff.jsDefinitionName].outcomes.push( + newStuff.dictTupleElmCode + ); + } } flushResults(); From ffa8275187954e1bd8b280ad73d8fc0041073cec Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sat, 1 Aug 2026 10:38:18 +0200 Subject: [PATCH 36/81] WIP use new API --- elm/src/Test/Runner/Node.elm | 134 +++++++++++++++-------------------- lib/Generate.js | 17 +---- 2 files changed, 59 insertions(+), 92 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index ddd307b7..167952aa 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,4 @@ -port module Test.Runner.Node exposing (check, run, TestProgram, PreviousRun) +port module Test.Runner.Node exposing (foo, run, TestProgram, PreviousRun) {-| @@ -8,7 +8,7 @@ port module Test.Runner.Node exposing (check, run, TestProgram, PreviousRun) Runs a test and outputs its results to the console. Exit code is 0 if tests passed and 2 if any failed. Returns 1 if something went wrong. -@docs check, run, TestProgram, PreviousRun +@docs foo, run, TestProgram, PreviousRun -} @@ -22,7 +22,7 @@ import Task import Test exposing (Test) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) import Test.Reporter.TestResults exposing (Outcome, TestResult, isFailure, outcomeFromExpectations) -import Test.Runner exposing (Runner, SeededRunners(..)) +import Test.Runner exposing (FuzzTest, Tests, UnitTest) import Test.Runner.JsMessage as JsMessage exposing (JsMessage(..)) import Time exposing (Posix) @@ -41,9 +41,9 @@ type alias InitArgs = , globs : List String , paths : List String , fuzzRuns : Int - , runners : SeededRunners + , tests : Tests , report : Report - , metadata : Metadata + , hashes : Hashes , previousRun : PreviousRun } @@ -55,31 +55,28 @@ type alias RunnerOptions = , globs : List String , paths : List String , processes : Int + , hashes : Hashes , previousRun : PreviousRun } type alias Model = - { available : Dict TestId Runner + { unitTests : Dict TestId UnitTest + , fuzzTests : Dict TestId FuzzTest , runInfo : RunInfo , testReporter : TestReporter , results : List ( TestId, TestResult ) , processes : Int , nextTestToRun : TestId , autoFail : Maybe String - , metadata : Metadata + , hashes : Hashes , previousRun : PreviousRun } -type alias Metadata = - Dict ( String, String ) MetadataItem - - -type alias MetadataItem = - { jsDefinitionName : String - , hash : String - } +type alias Hashes = + -- jsDefinitionName to hash + Dict String String type alias PreviousRun = @@ -98,7 +95,7 @@ type alias TestProgram = type Msg = Receive Decode.Value | Dispatch Posix - | Complete MetadataItem (List String) Outcome2 Posix Posix + | Complete String {- MetadataItem -} (List String) Outcome2 Posix Posix {-| The port names are prefixed to reduce the likelihood of the project @@ -384,39 +381,32 @@ sendBegin model = init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { processes, globs, paths, fuzzRuns, initialSeed, report, runners, metadata, previousRun } index = +init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, hashes, previousRun } index = let - { indexedRunners, autoFail } = - case runners of - Plain runnerList -> - { indexedRunners = List.indexedMap (\a b -> ( a, b )) runnerList - , autoFail = Nothing - } + autoFail = + case ( tests.seenOnly, tests.seenSkip ) of + ( False, False ) -> + Nothing - Only runnerList -> - { indexedRunners = List.indexedMap (\a b -> ( a, b )) runnerList - , autoFail = Just "Test.only was used" - } + ( True, False ) -> + Just "Test.only was used" - Skipping runnerList -> - { indexedRunners = List.indexedMap (\a b -> ( a, b )) runnerList - , autoFail = Just "Test.skip was used" - } + ( False, True ) -> + Just "Test.skip was used" - Invalid str -> - { indexedRunners = [] - , autoFail = Just str - } + ( True, True ) -> + Just "Test.only and Test.skip were used" testCount = - List.length indexedRunners + List.length tests.unitTests + List.length tests.fuzzTests testReporter = createReporter report model : Model model = - { available = Dict.fromList indexedRunners + { unitTests = toIndexedDict tests.unitTests + , fuzzTests = toIndexedDict tests.fuzzTests , runInfo = { testCount = testCount , globs = globs @@ -429,7 +419,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, runners, metadata , results = [] , testReporter = testReporter , autoFail = autoFail - , metadata = metadata + , hashes = hashes , previousRun = previousRun } @@ -453,7 +443,8 @@ failInit message report _ = let model : Model model = - { available = Dict.empty + { unitTests = Dict.empty + , fuzzTests = Dict.empty , runInfo = { testCount = 0 , globs = [] @@ -466,7 +457,7 @@ failInit message report _ = , results = [] , testReporter = createReporter report , autoFail = Nothing - , metadata = Dict.empty + , hashes = Dict.empty , previousRun = { fuzzRuns = 0 , initialSeed = 0 @@ -485,12 +476,17 @@ failInit message report _ = ( model, cmd ) -type alias TestWithMetadata = - { test : Test - , jsDefinitionName : String - , hash : String - , labels : Set String - } +toIndexedDict : List a -> Dict Int a +toIndexedDict list = + list + |> List.indexedMap Tuple.pair + |> Dict.fromList + + +foo : a -> String -> Maybe Test +foo value jsDefinitionName = + check value + |> Maybe.map (Test.Runner.tagTest jsDefinitionName) {-| The implementation of this function will be replaced in the generated JS @@ -499,56 +495,38 @@ with a version that returns `Just value` if `value` is a `Test`, otherwise `Noth If you rename or change this function you also need to update the regex that looks for it. -} -check : a -> String -> String -> Maybe TestWithMetadata +check : a -> Maybe Test check = checkHelperReplaceMe___ -checkHelperReplaceMe___ : a -> String -> String -> b -checkHelperReplaceMe___ _ _ _ = +checkHelperReplaceMe___ : a -> b +checkHelperReplaceMe___ _ = Debug.todo "The regex for replacing this Debug.todo in checkHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" {-| Run the tests. -} -run : RunnerOptions -> List ( String, List (Maybe TestWithMetadata) ) -> Program Int Model Msg -run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = +run : RunnerOptions -> List ( String, List (Maybe Test) ) -> Program Int Model Msg +run { runs, seed, report, globs, paths, processes, hashes, previousRun } possiblyTests = + -- TODO: Codegen the hashes. let - ( tests, metadata ) = + testsList = possiblyTests |> List.filterMap (\( moduleName, maybeModuleTests ) -> let - moduleTestsWithMetadata = - List.filterMap identity maybeModuleTests - moduleTests = - List.map .test moduleTestsWithMetadata + List.filterMap identity maybeModuleTests in if List.isEmpty moduleTests then Nothing else - Just - ( Test.describe moduleName moduleTests - , moduleTestsWithMetadata - |> List.concatMap - (\data -> - data.labels - |> Set.toList - |> List.map - (\label -> - ( ( moduleName, label ) - , { jsDefinitionName = data.jsDefinitionName, hash = data.hash } - ) - ) - ) - ) + Just (Test.describe moduleName moduleTests) ) - |> List.unzip - |> Tuple.mapSecond (List.concat >> Dict.fromList) in - if List.isEmpty tests then + if List.isEmpty testsList then Platform.worker { init = failInit (noTestsFoundError globs) report , update = \_ model -> ( model, Cmd.none ) @@ -557,8 +535,8 @@ run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = else let - runners = - Test.Runner.fromTest runs (Random.initialSeed seed) (Test.concat tests) + tests = + Test.Runner.toTests (Test.concat testsList) wrappedInit = init @@ -567,9 +545,9 @@ run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = , globs = globs , paths = paths , fuzzRuns = runs - , runners = runners + , tests = tests , report = report - , metadata = metadata + , hashes = hashes , previousRun = previousRun } in diff --git a/lib/Generate.js b/lib/Generate.js index 5be9a80a..856c2846 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -86,6 +86,7 @@ const hashPlaceholder = /__elm_test_hash__:([$_][$\w\u0080-\uffff]+)/g; * @returns { string } */ function makeHashPlaceholder(name) { + // TODO: Just code-gen the entire dict instead! return `__elm_test_hash__:${name}`; } @@ -117,18 +118,7 @@ function patch(hashes, content) { .replace(testVariantDefinition, '$&__elmTestSymbol: __elmTestSymbol, ') .replace( checkDefinition, - // Abuse `$elm_explorations$test$Test$Internal$duplicatedName` to get the test name(s). - // Usually, a `Test` has a single name (from `test`, `fuzz` or `describe`), but `Test.concat` - // does not have a name – instead we need to use the names of all the tests it concatenates - // (recursively). `$elm_explorations$test$Test$Internal$duplicatedName` returns the unique - // names when there are no duplicates – and there shouldn’t be, because that is not constructable. - ` -$1 = F3((test, jsDefinitionName, hash) => - test && test.__elmTestSymbol === __elmTestSymbol - ? $elm$core$Maybe$Just({ test, jsDefinitionName, hash, labels: $elm_explorations$test$Test$Internal$duplicatedName(_List_Cons(test, _List_Nil)).a }) - : $elm$core$Maybe$Nothing -); -`.trim() + '$1 = value => value && value.__elmTestSymbol === __elmTestSymbol ? $elm$core$Maybe$Just(value) : $elm$core$Maybe$Nothing;' ) // Simply remove the first occurrence of `console.warn`. This leaves the message string in parentheses behind, but that’s fine. .replace('console.warn', '') @@ -320,8 +310,7 @@ main = function makeModuleTuple(mod) { const list = mod.possiblyTests.map((test) => { const name = toCompiledJavaScriptName(mod.moduleName, test); - const newHash = makeHashPlaceholder(name); - return `Test.Runner.Node.check ${mod.moduleName}.${test} "${name}" "${newHash}"`; + return `Test.Runner.Node.foo ${mod.moduleName}.${test} "${name}"`; }); return ` From be3ba7c55d2cef290d2326f4ce8a6dc1e1c5eae7 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sat, 1 Aug 2026 22:38:48 +0200 Subject: [PATCH 37/81] Implement Debug.log and getHashes --- elm/src/Test/Runner/Node.elm | 80 ++++++++++++++++------------------- lib/Generate.js | 82 ++++++++++++++++++++---------------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 167952aa..35120621 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -43,7 +43,6 @@ type alias InitArgs = , fuzzRuns : Int , tests : Tests , report : Report - , hashes : Hashes , previousRun : PreviousRun } @@ -55,13 +54,12 @@ type alias RunnerOptions = , globs : List String , paths : List String , processes : Int - , hashes : Hashes , previousRun : PreviousRun } type alias Model = - { unitTests : Dict TestId UnitTest + { unitTests : List UnitTest , fuzzTests : Dict TestId FuzzTest , runInfo : RunInfo , testReporter : TestReporter @@ -69,16 +67,10 @@ type alias Model = , processes : Int , nextTestToRun : TestId , autoFail : Maybe String - , hashes : Hashes , previousRun : PreviousRun } -type alias Hashes = - -- jsDefinitionName to hash - Dict String String - - type alias PreviousRun = { fuzzRuns : Int , initialSeed : Int @@ -174,7 +166,7 @@ dispatch model startTime = Nothing -> let ( expectations, isFuzzTest, usedDebugLog ) = - detectFuzzTestAndDebugLog config.run + config.run () in { outcome = outcomeFromExpectations expectations , isFuzzTest = isFuzzTest @@ -198,23 +190,6 @@ lastTwoReversed list = Nothing -{-| The implementation of this function will be replaced in the generated JS -with a version that returns calls the passed function, and detects if it was -a fuzz test. - -If you rename or change this function you also need to update the regex that looks for it. - --} -detectFuzzTestAndDebugLog : (() -> a) -> ( a, Bool, Bool ) -detectFuzzTestAndDebugLog = - detectFuzzTestAndDebugLogHelperReplaceMe___ - - -detectFuzzTestAndDebugLogHelperReplaceMe___ : (() -> a) -> ( a, Bool, Bool ) -detectFuzzTestAndDebugLogHelperReplaceMe___ _ = - Debug.todo "The regex for replacing this Debug.todo in detectFuzzTestAndDebugLogHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" - - update : Msg -> Model -> ( Model, Cmd Msg ) update msg ({ testReporter } as model) = case msg of @@ -381,7 +356,7 @@ sendBegin model = init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, hashes, previousRun } index = +init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } index = let autoFail = case ( tests.seenOnly, tests.seenSkip ) of @@ -405,7 +380,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, hashes, pr model : Model model = - { unitTests = toIndexedDict tests.unitTests + { unitTests = tests.unitTests , fuzzTests = toIndexedDict tests.fuzzTests , runInfo = { testCount = testCount @@ -419,7 +394,6 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, hashes, pr , results = [] , testReporter = testReporter , autoFail = autoFail - , hashes = hashes , previousRun = previousRun } @@ -457,7 +431,6 @@ failInit message report _ = , results = [] , testReporter = createReporter report , autoFail = Nothing - , hashes = Dict.empty , previousRun = { fuzzRuns = 0 , initialSeed = 0 @@ -489,27 +462,49 @@ foo value jsDefinitionName = |> Maybe.map (Test.Runner.tagTest jsDefinitionName) -{-| The implementation of this function will be replaced in the generated JS -with a version that returns `Just value` if `value` is a `Test`, otherwise `Nothing`. - -If you rename or change this function you also need to update the regex that looks for it. - +{-| Returns `Just value` if `value` is a `Test`, otherwise `Nothing`. -} check : a -> Maybe Test check = - checkHelperReplaceMe___ + placeholderReplaceMe___ "check" -checkHelperReplaceMe___ : a -> b -checkHelperReplaceMe___ _ = - Debug.todo "The regex for replacing this Debug.todo in checkHelperReplaceMe___ with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" +{-| Returns all debug logs created since the beginning, +or last time this function was called. + +The `Bool` is set to `True` for fuzz tests, and pauses +`Debug.log` - logging is not supported for passing fuzz +tests. If the fuzz test fails, the failing run is run +again and that time we do collect logs. + +-} +getAndClearDebugLogs : Bool -> Decode.Value +getAndClearDebugLogs = + placeholderReplaceMe___ "getAndClearDebugLogs" + + +{-| Takes a `jsDefinitionName` and returns its hash. +-} +getHash : String -> String +getHash = + placeholderReplaceMe___ "getHash" + + +{-| The implementation of functions calling this one will be replaced in the generated JS +with versions that do something not normally possible in Elm. + +If you rename or change this function, or any function that calls it, you also need to update the regexes that looks for it. + +-} +placeholderReplaceMe___ : String -> a +placeholderReplaceMe___ name = + Debug.todo ("The regex for replacing this Debug.todo for '" ++ name ++ "' with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n") {-| Run the tests. -} run : RunnerOptions -> List ( String, List (Maybe Test) ) -> Program Int Model Msg -run { runs, seed, report, globs, paths, processes, hashes, previousRun } possiblyTests = - -- TODO: Codegen the hashes. +run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = let testsList = possiblyTests @@ -547,7 +542,6 @@ run { runs, seed, report, globs, paths, processes, hashes, previousRun } possibl , fuzzRuns = runs , tests = tests , report = report - , hashes = hashes , previousRun = previousRun } in diff --git a/lib/Generate.js b/lib/Generate.js index 836c14a1..d42ffd36 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -57,39 +57,33 @@ ${after} return hashes; } -// For older versions of elm-explorations/test we need to list every single -// variant of the `Test` type. To avoid having to update this regex if a new -// variant is added, newer versions of elm-explorations/test have prefixed all -// variants with `ElmTestVariant__` so we can match just on that. +// To avoid having to update this regex if a new variant is added, +// elm-explorations/test have prefixed all variants with `ElmTestVariant__`. // `\$?` is for the Lamdera compiler, where definitions sometimes end with a `$`. // See https://github.com/lamdera/compiler/pull/41#issuecomment-2725158568 const testVariantDefinition = - /^var\s+\$elm_explorations\$test\$Test\$Internal\$(?:ElmTestVariant__\w+|UnitTest|FuzzTest|Labeled|Skipped|Only|Batch)\$?\s*=\s*(?:\w+\(\s*)?function\s*\([\w, ]*\)\s*\{\s*return *\{/gm; - -const checkDefinition = - /^(var\s+\$author\$project\$Test\$Runner\$Node\$check)\s*=\s*\$author\$project\$Test\$Runner\$Node\$checkHelperReplaceMe___;?$/m; - -const detectFuzzTestAndDebugLogDefinition = - /^(var\s+\$author\$project\$Test\$Runner\$Node\$detectFuzzTestAndDebugLog)\s*=\s*\$author\$project\$Test\$Runner\$Node\$detectFuzzTestAndDebugLogHelperReplaceMe___;?$/m; - -const fuzzLoopDefinition = - /^var \$elm_explorations\$test\$Test\$Fuzz\$fuzzLoop\s*=\s*F\d\(\s*function\s*\([^()]*\)\s*\{/m; - -const debugLogDefinition = - /^var _Debug_log\s*=\s*F\d\(\s*function\s*\([^()]*\)\s*\{/m; - -// For the identifier, this uses the same regex as `REFERENCES_REGEX` in `Hash.js`. -const hashPlaceholder = /__elm_test_hash__:([$_][$\w\u0080-\uffff]+)/g; + /^var \$elm_explorations\$test\$Test\$Internal\$(?:ElmTestVariant__\w+)\$? = (?:F\d\(\s*)?function \([\w, ]*\) \{\s*return \{/gm; /** * @param { string } name - * @returns { string } + * @returns { RegExp } */ -function makeHashPlaceholder(name) { - // TODO: Just code-gen the entire dict instead! - return `__elm_test_hash__:${name}`; +function placeholderDefinition(name) { + return RegExp( + String.raw`^(var \$author\$project\$Test\$Runner\$Node\$${name}) = \$author\$project\$Test\$Runner\$Node\$placeholderReplaceMe___\('[^']+'\)`, + 'm' + ); } +const checkDefinition = placeholderDefinition('check'); +const getAndClearDebugLogsDefinition = placeholderDefinition( + 'getAndClearDebugLogs' +); +const getHashDefinition = placeholderDefinition('getHash'); + +const debugLogDefinition = + /^var _Debug_log = F2\(function \(tag, value\)\s*\{[^}]+\}\)/m; + /** * @param { string } moduleName * @param { string } valueName @@ -123,20 +117,36 @@ function patch(hashes, content) { // Simply remove the first occurrence of `console.warn`. This leaves the message string in parentheses behind, but that’s fine. .replace('console.warn', '') .replace( - detectFuzzTestAndDebugLogDefinition, + debugLogDefinition, ` -var _elmTestIsFuzzTest = false; -var _elmTestUsedDebugLog = false; -$1 = (f) => { - _elmTestIsFuzzTest = false; - _elmTestUsedDebugLog = false; - return _Utils_Tuple3(f(null), _elmTestIsFuzzTest, _elmTestUsedDebugLog); -} -`.trim() +var _Debug_logs = []; +var _Debug_logPaused = false; +var _Debug_logPausedMessage = 'For passing fuzz tests, Debug.log is not shown, since showing logs from lots of runs is pretty confusing. Tip: Use Debug.todo to fail a test from anywhere.'; +var _Debug_log = F2(function(tag, value) +{ + if (_Debug_logPaused) { + if (_Debug_logs.length === 0) { + _Debug_logs.push(_Debug_logPausedMessage); + } + } else { + _Debug_logs.push(tag + ': ' + _Debug_toString(value)); + } + return value; +}); + `.trim() + ) + .replace( + getAndClearDebugLogsDefinition, + '$1 = pause => { var logs = _Json_wrap(_Debug_logs); _Debug_logs = []; _Debug_logPaused = paused; return logs; }' + ) + .replace( + getHashDefinition, + `var __elmTestHashes = ${JSON.stringify( + hashes, + null, + 2 + )};\n$1 = name => __elmTestHashes[name]` ) - .replace(fuzzLoopDefinition, '$& _elmTestIsFuzzTest = true;') - .replace(debugLogDefinition, '$& _elmTestUsedDebugLog = true;') - .replace(hashPlaceholder, (_match, name) => hashes[name]) ); } From e9e0e8be95f952e1f65335f4db82e268cad51ac0 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sat, 1 Aug 2026 23:23:37 +0200 Subject: [PATCH 38/81] Work on running unit tests --- elm/src/Test/Runner/JsMessage.elm | 11 +- elm/src/Test/Runner/Node.elm | 190 ++++++++++++------------------ 2 files changed, 86 insertions(+), 115 deletions(-) diff --git a/elm/src/Test/Runner/JsMessage.elm b/elm/src/Test/Runner/JsMessage.elm index a6a5c5bd..d106375d 100644 --- a/elm/src/Test/Runner/JsMessage.elm +++ b/elm/src/Test/Runner/JsMessage.elm @@ -4,7 +4,9 @@ import Json.Decode as Decode exposing (Decoder) type JsMessage - = Summary Float Int (List ( List String, String )) + = RunUnitTests + | RunFuzzTest Int + | Summary Float Int (List ( List String, String )) decoder : Decoder JsMessage @@ -16,6 +18,13 @@ decoder = decodeMessageFromType : String -> Decoder JsMessage decodeMessageFromType messageType = case messageType of + "RunUnitTests" -> + Decode.succeed RunUnitTests + + "RunFuzzTest" -> + Decode.map RunFuzzTest + (Decode.field "testId" Decode.int) + "SUMMARY" -> Decode.map3 Summary (Decode.field "duration" Decode.float) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 35120621..8743e0c3 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -22,7 +22,8 @@ import Task import Test exposing (Test) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) import Test.Reporter.TestResults exposing (Outcome, TestResult, isFailure, outcomeFromExpectations) -import Test.Runner exposing (FuzzTest, Tests, UnitTest) +import Test.Runner exposing (FuzzTest, FuzzTestExpectation(..), Tests, UnitTest, UnitTestExpectation(..)) +import Test.Runner.Failure exposing (Reason) import Test.Runner.JsMessage as JsMessage exposing (JsMessage(..)) import Time exposing (Posix) @@ -35,6 +36,14 @@ type alias TestId = Int +type alias JsDefinitionName = + String + + +type alias DebugLogs = + Decode.Value + + type alias InitArgs = { initialSeed : Int , processes : Int @@ -63,9 +72,7 @@ type alias Model = , fuzzTests : Dict TestId FuzzTest , runInfo : RunInfo , testReporter : TestReporter - , results : List ( TestId, TestResult ) , processes : Int - , nextTestToRun : TestId , autoFail : Maybe String , previousRun : PreviousRun } @@ -74,7 +81,7 @@ type alias Model = type alias PreviousRun = { fuzzRuns : Int , initialSeed : Int - , fingerprints : Dict String Fingerprints + , cachedTests : Dict JsDefinitionName CachedTests } @@ -86,8 +93,8 @@ type alias TestProgram = type Msg = Receive Decode.Value - | Dispatch Posix - | Complete String {- MetadataItem -} (List String) Outcome2 Posix Posix + | DispatchUnitTest Posix + | Complete String (List String) UnitTestExpectation DebugLogs (List UnitTest) Posix Posix {-| The port names are prefixed to reduce the likelihood of the project @@ -99,95 +106,58 @@ port elmTestPort__send : Decode.Value -> Cmd msg port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg -type alias Fingerprints = +type alias CachedTests = { hash : String - , outcomes : Dict (List String) Outcome2 - } + -- As an optimization, passing unit tests without debug logs are not stored. + , unitTests : Dict (List String) ( UnitTestExpectation, DebugLogs ) -type alias Outcome2 = - { outcome : Outcome - , isFuzzTest : Bool - , usedDebugLog : Bool + -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. + , fuzzTests : Dict (List String) ( FuzzTestExpectation, DebugLogs ) } -dispatch : Model -> Posix -> Cmd Msg -dispatch model startTime = - case Dict.get model.nextTestToRun model.available of - Nothing -> +noDebugLogs : DebugLogs +noDebugLogs = + Encode.list never [] + + +dispatchUnitTest : Model -> Posix -> Cmd Msg +dispatchUnitTest model startTime = + case model.unitTests of + [] -> -- We're finished! Nothing left to run. + -- TODO: Send new message: finished but no data sendResults True model.testReporter model.results - Just config -> + unitTest :: remainingUnitTests -> let - metadata = - case lastTwoReversed config.labels |> Maybe.andThen (\key -> Dict.get key model.metadata) of - Just metadata_ -> - metadata_ - - -- A `Test.todo` not nested under any `Test.describe` ends up here (`config.labels` only - -- contains _one_ label – the one added automatically to each module). - -- Luckily, there is not really much speed gain to caching `Test.todo`s. - Nothing -> - { jsDefinitionName = "", hash = "" } + hash = + getHash unitTest.tag - maybeCachedOutcome = - Dict.get metadata.jsDefinitionName model.previousRun.fingerprints + maybeCached = + Dict.get unitTest.tag model.previousRun.cachedTests |> Maybe.andThen - (\fingerprints -> - if metadata.hash == fingerprints.hash then - Dict.get config.labels fingerprints.outcomes - |> Maybe.andThen - (\outcome_ -> - if - not outcome_.usedDebugLog - && (not outcome_.isFuzzTest - || ((model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) - && (model.runInfo.initialSeed == model.previousRun.initialSeed) - ) - ) - then - Just outcome_ - - else - Nothing - ) + (\cachedTests -> + if hash == cachedTests.hash then + Dict.get unitTest.labels cachedTests.unitTests + -- As an optimization, passing unit tests without debug logs are not stored. + |> Maybe.withDefault ( UnitTestPass, noDebugLogs ) else Nothing ) - outcome = - case maybeCachedOutcome of - Just outcome_ -> - outcome_ + ( expectation, debugLogs ) = + case maybeCached of + Just cached -> + cached Nothing -> - let - ( expectations, isFuzzTest, usedDebugLog ) = - config.run () - in - { outcome = outcomeFromExpectations expectations - , isFuzzTest = isFuzzTest - , usedDebugLog = usedDebugLog - } + ( unitTest.thunk (), getAndClearDebugLogs False ) in Time.now - |> Task.perform (Complete metadata config.labels outcome startTime) - - -lastTwoReversed : List a -> Maybe ( a, a ) -lastTwoReversed list = - case list of - [ a, b ] -> - Just ( b, a ) - - _ :: rest -> - lastTwoReversed rest - - _ -> - Nothing + |> Task.perform (Complete hash unitTest.labels expectation debugLogs remainingUnitTests startTime) update : Msg -> Model -> ( Model, Cmd Msg ) @@ -195,6 +165,15 @@ update msg ({ testReporter } as model) = case msg of Receive val -> case Decode.decodeValue JsMessage.decoder val of + Ok RunUnitTests -> + ( model + , Task.perform DispatchUnitTest Time.now + ) + + Ok (RunFuzzTest testId) -> + -- TODO: Run fuzz test + ( model, Cmd.none ) + Ok (Summary duration failed todos) -> let testCount = @@ -242,10 +221,10 @@ update msg ({ testReporter } as model) = in ( model, cmd ) - Dispatch startTime -> - ( model, dispatch model startTime ) + DispatchUnitTest startTime -> + ( model, dispatchUnitTest model startTime ) - Complete metadata labels outcome2 startTime endTime -> + Complete hash labels expectation debugLogs remainingUnitTests startTime endTime -> let duration = Time.posixToMillis endTime - Time.posixToMillis startTime @@ -268,32 +247,16 @@ update msg ({ testReporter } as model) = isFinished = nextTestToRun >= model.runInfo.testCount in - if isFinished || isFailure outcome2.outcome then - let - cmd = - sendResults isFinished testReporter results - in - if isFinished then - -- Don't bother updating the model, since we're done - ( model, cmd ) - - else - -- Clear out the results, now that we've flushed them. - ( { model | nextTestToRun = nextTestToRun, results = [] } - , Cmd.batch - [ cmd - , Task.perform Dispatch Time.now - ] - ) - - else - ( { model | nextTestToRun = nextTestToRun, results = results } - , Task.perform Dispatch Time.now - ) + ( { model | unitTests = remainingUnitTests } + , Cmd.batch + [ cmd + , sendResults isFinished testReporter results + ] + ) -sendResults : Bool -> TestReporter -> List ( TestId, TestResult ) -> Cmd msg -sendResults isFinished testReporter results = +sendResults : TestReporter -> List ( TestId, TestResult ) -> Cmd msg +sendResults testReporter results = let typeStr = if isFinished then @@ -338,11 +301,6 @@ sendResults isFinished testReporter results = sendBegin : Model -> Cmd msg sendBegin model = let - baseFields = - [ ( "type", Encode.string "BEGIN" ) - , ( "testCount", Encode.int model.runInfo.testCount ) - ] - extraFields = case model.testReporter.reportBegin model.runInfo of Just report -> @@ -350,8 +308,14 @@ sendBegin model = Nothing -> [] + + fields = + ( "type", Encode.string "BEGIN" ) + :: ( "testCount", Encode.int model.runInfo.testCount ) + :: ( "fuzzTests", Encode.list Encode.int (Dict.keys model.fuzzTests) ) + :: extraFields in - Encode.object (baseFields ++ extraFields) + Encode.object fields |> elmTestPort__send @@ -390,7 +354,6 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRu , initialSeed = initialSeed } , processes = processes - , nextTestToRun = index , results = [] , testReporter = testReporter , autoFail = autoFail @@ -398,7 +361,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRu } cmd = - Task.perform Dispatch Time.now + Task.perform DispatchUnitTest Time.now in ( model , Cmd.batch @@ -417,7 +380,7 @@ failInit message report _ = let model : Model model = - { unitTests = Dict.empty + { unitTests = [] , fuzzTests = Dict.empty , runInfo = { testCount = 0 @@ -427,7 +390,6 @@ failInit message report _ = , initialSeed = 0 } , processes = 0 - , nextTestToRun = 0 , results = [] , testReporter = createReporter report , autoFail = Nothing @@ -456,7 +418,7 @@ toIndexedDict list = |> Dict.fromList -foo : a -> String -> Maybe Test +foo : a -> JsDefinitionName -> Maybe Test foo value jsDefinitionName = check value |> Maybe.map (Test.Runner.tagTest jsDefinitionName) @@ -478,14 +440,14 @@ tests. If the fuzz test fails, the failing run is run again and that time we do collect logs. -} -getAndClearDebugLogs : Bool -> Decode.Value +getAndClearDebugLogs : Bool -> DebugLogs getAndClearDebugLogs = placeholderReplaceMe___ "getAndClearDebugLogs" {-| Takes a `jsDefinitionName` and returns its hash. -} -getHash : String -> String +getHash : JsDefinitionName -> String getHash = placeholderReplaceMe___ "getHash" From d56480e0b1e3fe94a46bddf4de4fd121241a022d Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 01:04:18 +0200 Subject: [PATCH 39/81] Running UnitTest compiles --- elm/elm.json | 2 +- elm/src/Test/Reporter/JUnit.elm | 7 +- elm/src/Test/Reporter/Json.elm | 2 +- elm/src/Test/Reporter/TestResults.elm | 5 +- elm/src/Test/Runner/JsMessage.elm | 42 ---- elm/src/Test/Runner/Node.elm | 266 +++++++++----------------- elm/src/Test/Runner/Ports.elm | 120 ++++++++++++ lib/Generate.js | 5 + 8 files changed, 218 insertions(+), 231 deletions(-) delete mode 100644 elm/src/Test/Runner/JsMessage.elm create mode 100644 elm/src/Test/Runner/Ports.elm diff --git a/elm/elm.json b/elm/elm.json index 08e5084d..b83f716f 100644 --- a/elm/elm.json +++ b/elm/elm.json @@ -9,12 +9,12 @@ "elm/core": "1.0.5", "elm/json": "1.1.3", "elm/random": "1.0.0", - "elm/time": "1.0.0", "elm-explorations/test": "2.2.1" }, "indirect": { "elm/bytes": "1.0.8", "elm/html": "1.0.0", + "elm/time": "1.0.0", "elm/virtual-dom": "1.0.3" } }, diff --git a/elm/src/Test/Reporter/JUnit.elm b/elm/src/Test/Reporter/JUnit.elm index 66077675..689d58c2 100644 --- a/elm/src/Test/Reporter/JUnit.elm +++ b/elm/src/Test/Reporter/JUnit.elm @@ -84,9 +84,9 @@ formatClassAndName labels = ( "", "" ) -encodeDuration : Int -> Value +encodeDuration : Float -> Value encodeDuration time = - (toFloat time / 1000) + (time / 1000) |> String.fromFloat |> Encode.string @@ -119,9 +119,6 @@ encodeExtraFailure _ = } , NoDistribution ) - , jsDefinitionName = "" - , isFuzzTest = False - , usedDebugLog = False } diff --git a/elm/src/Test/Reporter/Json.elm b/elm/src/Test/Reporter/Json.elm index 643076d9..06b8d5ac 100644 --- a/elm/src/Test/Reporter/Json.elm +++ b/elm/src/Test/Reporter/Json.elm @@ -28,7 +28,7 @@ reportComplete { duration, labels, outcome } = , ( "labels", encodeLabels labels ) , ( "failures", Encode.list identity (encodeFailures outcome) ) , ( "distributionReports", Encode.list identity (encodeDistributionReports outcome) ) - , ( "duration", Encode.string <| String.fromInt duration ) + , ( "duration", Encode.string <| String.fromInt (round duration) ) ] diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index bb035fab..5380d4bf 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -22,10 +22,7 @@ type Outcome type alias TestResult = { labels : List String , outcome : Outcome - , duration : Int -- in milliseconds - , jsDefinitionName : String - , isFuzzTest : Bool - , usedDebugLog : Bool + , duration : Float -- in milliseconds } diff --git a/elm/src/Test/Runner/JsMessage.elm b/elm/src/Test/Runner/JsMessage.elm deleted file mode 100644 index d106375d..00000000 --- a/elm/src/Test/Runner/JsMessage.elm +++ /dev/null @@ -1,42 +0,0 @@ -module Test.Runner.JsMessage exposing (JsMessage(..), decoder) - -import Json.Decode as Decode exposing (Decoder) - - -type JsMessage - = RunUnitTests - | RunFuzzTest Int - | Summary Float Int (List ( List String, String )) - - -decoder : Decoder JsMessage -decoder = - Decode.field "type" Decode.string - |> Decode.andThen decodeMessageFromType - - -decodeMessageFromType : String -> Decoder JsMessage -decodeMessageFromType messageType = - case messageType of - "RunUnitTests" -> - Decode.succeed RunUnitTests - - "RunFuzzTest" -> - Decode.map RunFuzzTest - (Decode.field "testId" Decode.int) - - "SUMMARY" -> - Decode.map3 Summary - (Decode.field "duration" Decode.float) - (Decode.field "failures" Decode.int) - (Decode.field "todos" (Decode.list todoDecoder)) - - _ -> - Decode.fail ("Unrecognized message type: " ++ messageType) - - -todoDecoder : Decoder ( List String, String ) -todoDecoder = - Decode.map2 (\a b -> ( a, b )) - (Decode.field "labels" (Decode.list Decode.string)) - (Decode.field "todo" Decode.string) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 8743e0c3..ced85bec 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,4 @@ -port module Test.Runner.Node exposing (foo, run, TestProgram, PreviousRun) +module Test.Runner.Node exposing (foo, run, TestProgram, PreviousRun) {-| @@ -20,12 +20,12 @@ import Random import Set exposing (Set) import Task import Test exposing (Test) +import Test.Distribution exposing (DistributionReport(..)) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) -import Test.Reporter.TestResults exposing (Outcome, TestResult, isFailure, outcomeFromExpectations) +import Test.Reporter.TestResults exposing (Outcome(..), TestResult, isFailure, outcomeFromExpectations) import Test.Runner exposing (FuzzTest, FuzzTestExpectation(..), Tests, UnitTest, UnitTestExpectation(..)) -import Test.Runner.Failure exposing (Reason) -import Test.Runner.JsMessage as JsMessage exposing (JsMessage(..)) -import Time exposing (Posix) +import Test.Runner.Failure exposing (Reason(..)) +import Test.Runner.Ports as Ports exposing (JsMessage(..)) @@ -68,7 +68,7 @@ type alias RunnerOptions = type alias Model = - { unitTests : List UnitTest + { unitTests : Dict TestId UnitTest , fuzzTests : Dict TestId FuzzTest , runInfo : RunInfo , testReporter : TestReporter @@ -92,18 +92,7 @@ type alias TestProgram = type Msg - = Receive Decode.Value - | DispatchUnitTest Posix - | Complete String (List String) UnitTestExpectation DebugLogs (List UnitTest) Posix Posix - - -{-| The port names are prefixed to reduce the likelihood of the project -having a port with the same name, which is a compile error. --} -port elmTestPort__send : Decode.Value -> Cmd msg - - -port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg + = Receive (Result Decode.Error JsMessage) type alias CachedTests = @@ -122,59 +111,93 @@ noDebugLogs = Encode.list never [] -dispatchUnitTest : Model -> Posix -> Cmd Msg -dispatchUnitTest model startTime = - case model.unitTests of - [] -> - -- We're finished! Nothing left to run. - -- TODO: Send new message: finished but no data - sendResults True model.testReporter model.results +dispatchUnitTest : TestId -> Model -> ( Model, Cmd Msg ) +dispatchUnitTest testId model = + case Dict.get testId model.unitTests of + Nothing -> + ( model + , Ports.sendError ("Unit test not found: " ++ String.fromInt testId) + ) - unitTest :: remainingUnitTests -> + Just unitTest -> let + jsDefinitionName = + unitTest.tag + hash = - getHash unitTest.tag + getHash jsDefinitionName maybeCached = - Dict.get unitTest.tag model.previousRun.cachedTests + Dict.get jsDefinitionName model.previousRun.cachedTests |> Maybe.andThen (\cachedTests -> if hash == cachedTests.hash then Dict.get unitTest.labels cachedTests.unitTests -- As an optimization, passing unit tests without debug logs are not stored. |> Maybe.withDefault ( UnitTestPass, noDebugLogs ) + |> Just else Nothing ) - ( expectation, debugLogs ) = + ( ( expectation, duration ), debugLogs ) = case maybeCached of - Just cached -> - cached + Just ( expectation_, debugLogs_ ) -> + ( ( expectation_, 0 ), debugLogs_ ) Nothing -> - ( unitTest.thunk (), getAndClearDebugLogs False ) + ( runWithDuration unitTest.thunk, getAndClearDebugLogs False ) + + outcome = + case expectation of + UnitTestPass -> + Passed NoDistribution + + UnitTestFail { description, reason } -> + if reason == TODO then + Todo description + + else + Failed + ( { given = Nothing + , description = description + , reason = reason + } + , NoDistribution + ) + + result : TestResult + result = + { labels = unitTest.labels + , outcome = outcome + , duration = duration + } + + report = + model.testReporter.reportComplete result + + expectationElmCode = + Debug.toString expectation in - Time.now - |> Task.perform (Complete hash unitTest.labels expectation debugLogs remainingUnitTests startTime) + ( model + , Ports.sendResult testId jsDefinitionName unitTest.labels expectationElmCode debugLogs report + ) update : Msg -> Model -> ( Model, Cmd Msg ) update msg ({ testReporter } as model) = case msg of - Receive val -> - case Decode.decodeValue JsMessage.decoder val of - Ok RunUnitTests -> - ( model - , Task.perform DispatchUnitTest Time.now - ) + Receive (Ok jsMessage) -> + case jsMessage of + RunUnitTest testId -> + dispatchUnitTest testId model - Ok (RunFuzzTest testId) -> + RunFuzzTest testId -> -- TODO: Run fuzz test ( model, Cmd.none ) - Ok (Summary duration failed todos) -> + Summary duration failed todos -> let testCount = model.runInfo.testCount @@ -201,122 +224,12 @@ update msg ({ testReporter } as model) = 3 cmd = - Encode.object - [ ( "type", Encode.string "SUMMARY" ) - , ( "exitCode", Encode.int exitCode ) - , ( "message", summary ) - ] - |> elmTestPort__send + Ports.sendSummary exitCode summary in ( model, cmd ) - Err err -> - let - cmd = - Encode.object - [ ( "type", Encode.string "ERROR" ) - , ( "message", Encode.string (Decode.errorToString err) ) - ] - |> elmTestPort__send - in - ( model, cmd ) - - DispatchUnitTest startTime -> - ( model, dispatchUnitTest model startTime ) - - Complete hash labels expectation debugLogs remainingUnitTests startTime endTime -> - let - duration = - Time.posixToMillis endTime - Time.posixToMillis startTime - - results = - ( model.nextTestToRun - , { labels = labels - , outcome = outcome2.outcome - , duration = duration - , jsDefinitionName = metadata.jsDefinitionName - , isFuzzTest = outcome2.isFuzzTest - , usedDebugLog = outcome2.usedDebugLog - } - ) - :: model.results - - nextTestToRun = - model.nextTestToRun + model.processes - - isFinished = - nextTestToRun >= model.runInfo.testCount - in - ( { model | unitTests = remainingUnitTests } - , Cmd.batch - [ cmd - , sendResults isFinished testReporter results - ] - ) - - -sendResults : TestReporter -> List ( TestId, TestResult ) -> Cmd msg -sendResults testReporter results = - let - typeStr = - if isFinished then - "FINISHED" - - else - "RESULTS" - - addToKeyValues ( testId, result ) list = - -- These are coming in in reverse order. Doing a foldl with :: - -- means we reverse the list again, while also doing the conversion! - ( String.fromInt testId, testReporter.reportComplete result ) :: list - - encodeNewStuff ( _, result ) = - let - dictTuple : ( List String, Outcome2 ) - dictTuple = - ( result.labels - , { outcome = result.outcome - , isFuzzTest = result.isFuzzTest - , usedDebugLog = result.usedDebugLog - } - ) - in - Encode.object - [ ( "jsDefinitionName", Encode.string result.jsDefinitionName ) - , ( "dictTupleElmCode", Encode.string (Debug.toString dictTuple) ) - ] - in - Encode.object - [ ( "type", Encode.string typeStr ) - , ( "results" - , results - |> List.foldl addToKeyValues [] - |> Encode.object - ) - , ( "newStuff", Encode.list encodeNewStuff results ) - ] - |> elmTestPort__send - - -sendBegin : Model -> Cmd msg -sendBegin model = - let - extraFields = - case model.testReporter.reportBegin model.runInfo of - Just report -> - [ ( "message", report ) ] - - Nothing -> - [] - - fields = - ( "type", Encode.string "BEGIN" ) - :: ( "testCount", Encode.int model.runInfo.testCount ) - :: ( "fuzzTests", Encode.list Encode.int (Dict.keys model.fuzzTests) ) - :: extraFields - in - Encode.object fields - |> elmTestPort__send + Receive (Err err) -> + ( model, Ports.sendError (Decode.errorToString err) ) init : InitArgs -> Int -> ( Model, Cmd Msg ) @@ -344,7 +257,7 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRu model : Model model = - { unitTests = tests.unitTests + { unitTests = toIndexedDict tests.unitTests , fuzzTests = toIndexedDict tests.fuzzTests , runInfo = { testCount = testCount @@ -354,24 +267,21 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRu , initialSeed = initialSeed } , processes = processes - , results = [] , testReporter = testReporter , autoFail = autoFail , previousRun = previousRun } - - cmd = - Task.perform DispatchUnitTest Time.now in ( model - , Cmd.batch - [ cmd - , if index == 0 then - sendBegin model - - else - Cmd.none - ] + -- TODO: `index` doesn't really make sense anymore. + , if index == 0 then + Ports.sendBegin + (Dict.size model.unitTests) + (Dict.size model.fuzzTests) + (model.testReporter.reportBegin model.runInfo) + + else + Cmd.none ) @@ -380,7 +290,7 @@ failInit message report _ = let model : Model model = - { unitTests = [] + { unitTests = Dict.empty , fuzzTests = Dict.empty , runInfo = { testCount = 0 @@ -390,23 +300,18 @@ failInit message report _ = , initialSeed = 0 } , processes = 0 - , results = [] , testReporter = createReporter report , autoFail = Nothing , previousRun = { fuzzRuns = 0 , initialSeed = 0 - , fingerprints = Dict.empty + , cachedTests = Dict.empty } } cmd = - Encode.object - [ ( "type", Encode.string "SUMMARY" ) - , ( "exitCode", Encode.int 1 ) - , ( "message", Encode.string message ) - ] - |> elmTestPort__send + -- TODO: This isn't using the reporter? How does that work? + Ports.sendSummary 1 (Encode.string message) in ( model, cmd ) @@ -452,6 +357,11 @@ getHash = placeholderReplaceMe___ "getHash" +runWithDuration : (() -> a) -> ( a, Float ) +runWithDuration = + placeholderReplaceMe___ "runWithDuration" + + {-| The implementation of functions calling this one will be replaced in the generated JS with versions that do something not normally possible in Elm. @@ -510,7 +420,7 @@ run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = Platform.worker { init = wrappedInit , update = update - , subscriptions = \_ -> elmTestPort__receive Receive + , subscriptions = \_ -> Ports.receive Receive } diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm new file mode 100644 index 00000000..80c51029 --- /dev/null +++ b/elm/src/Test/Runner/Ports.elm @@ -0,0 +1,120 @@ +port module Test.Runner.Ports exposing (JsMessage(..), receive, sendBegin, sendError, sendResult, sendSummary) + +import Json.Decode as Decode exposing (Decoder) +import Json.Encode as Encode + + +{-| The port names are prefixed to reduce the likelihood of the project +having a port with the same name, which is a compile error. +-} +port elmTestPort__send : Decode.Value -> Cmd msg + + +port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg + + +sendBegin : Int -> Int -> Maybe Decode.Value -> Cmd msg +sendBegin unitTests fuzzTests maybeReport = + let + extraFields = + case maybeReport of + Just report -> + -- Test reporter specific: + [ ( "message", report ) ] + + Nothing -> + [] + in + elmTestPort__send + (Encode.object + (( "type", Encode.string "BEGIN" ) + :: ( "unitTests", Encode.int unitTests ) + :: ( "fuzzTests", Encode.int fuzzTests ) + :: extraFields + ) + ) + + +sendResult : Int -> String -> List String -> String -> Decode.Value -> Decode.Value -> Cmd msg +sendResult testId jsDefinitionName labels expectationElmCode debugLogs report = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "RESULT" ) + , ( "testId", Encode.int testId ) + , ( "jsDefinitionName", Encode.string jsDefinitionName ) + , ( "labels", Encode.list Encode.string labels ) + , ( "expectationElmCode", Encode.string expectationElmCode ) + , ( "debugLogs", debugLogs ) + + -- Test reporter specific: + , ( "message", report ) + ] + ) + + +sendSummary : Int -> Decode.Value -> Cmd msg +sendSummary exitCode summary = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "SUMMARY" ) + , ( "exitCode", Encode.int exitCode ) + + -- Test reporter specific: + , ( "message", summary ) + ] + ) + + +sendError : String -> Cmd msg +sendError message = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "ERROR" ) + , ( "message", Encode.string message ) + ] + ) + + +type JsMessage + = RunUnitTest Int + | RunFuzzTest Int + | Summary Float Int (List ( List String, String )) + + +decoder : Decoder JsMessage +decoder = + Decode.field "type" Decode.string + |> Decode.andThen decodeMessageFromType + + +decodeMessageFromType : String -> Decoder JsMessage +decodeMessageFromType messageType = + case messageType of + "RunUnitTest" -> + Decode.map RunUnitTest + (Decode.field "testId" Decode.int) + + "RunFuzzTest" -> + Decode.map RunFuzzTest + (Decode.field "testId" Decode.int) + + "SUMMARY" -> + Decode.map3 Summary + (Decode.field "duration" Decode.float) + (Decode.field "failures" Decode.int) + (Decode.field "todos" (Decode.list todoDecoder)) + + _ -> + Decode.fail ("Unrecognized message type: " ++ messageType) + + +todoDecoder : Decoder ( List String, String ) +todoDecoder = + Decode.map2 (\a b -> ( a, b )) + (Decode.field "labels" (Decode.list Decode.string)) + (Decode.field "todo" Decode.string) + + +receive : (Result Decode.Error JsMessage -> msg) -> Sub msg +receive toMsg = + elmTestPort__receive (Decode.decodeValue decoder >> toMsg) diff --git a/lib/Generate.js b/lib/Generate.js index d42ffd36..69922294 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -80,6 +80,7 @@ const getAndClearDebugLogsDefinition = placeholderDefinition( 'getAndClearDebugLogs' ); const getHashDefinition = placeholderDefinition('getHash'); +const runWithDurationDefinition = placeholderDefinition('runWithDuration'); const debugLogDefinition = /^var _Debug_log = F2\(function \(tag, value\)\s*\{[^}]+\}\)/m; @@ -147,6 +148,10 @@ var _Debug_log = F2(function(tag, value) 2 )};\n$1 = name => __elmTestHashes[name]` ) + .replace( + runWithDurationDefinition, + '$1 = thunk => { var t = performance.now(); return _Utils_Tuple2(thunk(null), performance.now() - t); }' + ) ); } From 4e6eddffe2b5b4dad2eb25d0eb8468a68221f252 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 01:55:00 +0200 Subject: [PATCH 40/81] Fuzz tests compile --- elm/src/Test/Runner/Node.elm | 157 ++++++++++++++++++++++++++++++++-- elm/src/Test/Runner/Ports.elm | 18 +++- 2 files changed, 165 insertions(+), 10 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index ced85bec..2dbe1252 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -111,6 +111,11 @@ noDebugLogs = Encode.list never [] +isEmptyDebugLogs : DebugLogs -> Bool +isEmptyDebugLogs debugLogs = + Decode.decodeValue (Decode.field "length" Decode.int) debugLogs == Ok 0 + + dispatchUnitTest : TestId -> Model -> ( Model, Cmd Msg ) dispatchUnitTest testId model = case Dict.get testId model.unitTests of @@ -132,10 +137,13 @@ dispatchUnitTest testId model = |> Maybe.andThen (\cachedTests -> if hash == cachedTests.hash then - Dict.get unitTest.labels cachedTests.unitTests + case Dict.get unitTest.labels cachedTests.unitTests of -- As an optimization, passing unit tests without debug logs are not stored. - |> Maybe.withDefault ( UnitTestPass, noDebugLogs ) - |> Just + Nothing -> + Just ( UnitTestPass, noDebugLogs ) + + cached -> + cached else Nothing @@ -178,13 +186,151 @@ dispatchUnitTest testId model = model.testReporter.reportComplete result expectationElmCode = - Debug.toString expectation + if expectation == UnitTestPass && isEmptyDebugLogs debugLogs then + Nothing + + else + Just (Debug.toString expectation) in ( model , Ports.sendResult testId jsDefinitionName unitTest.labels expectationElmCode debugLogs report ) +dispatchFuzzTest : TestId -> Model -> ( Model, Cmd Msg ) +dispatchFuzzTest testId model = + case Dict.get testId model.fuzzTests of + Nothing -> + ( model + , Ports.sendError ("Fuzz test not found: " ++ String.fromInt testId) + ) + + Just fuzzTest -> + let + jsDefinitionName = + fuzzTest.tag + + hash = + getHash jsDefinitionName + + ( fuzzerInts, maybeCached ) = + case Dict.get jsDefinitionName model.previousRun.cachedTests of + Nothing -> + ( [], Nothing ) + + Just cachedTests -> + let + canUseCached = + (hash == cachedTests.hash) + && (model.runInfo.initialSeed == model.previousRun.initialSeed) + -- If the fuzz tests specifies its own number of runs and the hash is the same, + -- then the number of runs must be unchanged. + && (fuzzTest.runs /= Nothing || model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) + in + case Dict.get fuzzTest.labels cachedTests.fuzzTests of + -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. + Nothing -> + ( [] + , if canUseCached then + Just ( FuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) + + else + Nothing + ) + + (Just ( expectation_, debugLogs_ )) as cached -> + let + fuzzerInts_ = + case expectation_ of + FuzzTestPass _ -> + [] + + FuzzTestFail data -> + data.fuzzerInts + in + ( fuzzerInts_ + , if canUseCached then + cached + + else + Nothing + ) + + ( ( expectation, duration ), debugLogs ) = + case maybeCached of + Just ( expectation_, debugLogs_ ) -> + ( ( expectation_, 0 ), debugLogs_ ) + + Nothing -> + let + seed = + Random.initialSeed model.runInfo.initialSeed + in + getAndClearDebugLogs True + |> (\_ -> + let + ( expectation_, duration_ ) = + runWithDuration (\() -> fuzzTest.thunk seed model.runInfo.fuzzRuns fuzzerInts) + in + case expectation_ of + FuzzTestPass data -> + ( ( expectation_, duration_ ) + , getAndClearDebugLogs False + ) + + FuzzTestFail data -> + let + newDebugLogs = + getAndClearDebugLogs False + |> (\_ -> + data.rerunFailure () + |> (\() -> getAndClearDebugLogs False) + ) + in + ( ( expectation_, duration_ ) + , newDebugLogs + ) + ) + + outcome = + case expectation of + FuzzTestPass { distributionReport } -> + Passed distributionReport + + FuzzTestFail { given, description, reason, distributionReport } -> + Failed + ( { given = given + , description = description + , reason = reason + } + , distributionReport + ) + + result : TestResult + result = + { labels = fuzzTest.labels + , outcome = outcome + , duration = duration + } + + report = + model.testReporter.reportComplete result + + expectationElmCode = + if expectation == FuzzTestPass { distributionReport = NoDistribution } && isEmptyDebugLogs debugLogs then + Nothing + + else + Debug.toString expectation + -- For `rerunFailure`: + |> String.replace "" "identity" + |> Just + in + ( model + , Ports.sendResult testId jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report + ) + + update : Msg -> Model -> ( Model, Cmd Msg ) update msg ({ testReporter } as model) = case msg of @@ -194,8 +340,7 @@ update msg ({ testReporter } as model) = dispatchUnitTest testId model RunFuzzTest testId -> - -- TODO: Run fuzz test - ( model, Cmd.none ) + dispatchFuzzTest testId model Summary duration failed todos -> let diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm index 80c51029..4286a4dd 100644 --- a/elm/src/Test/Runner/Ports.elm +++ b/elm/src/Test/Runner/Ports.elm @@ -35,7 +35,7 @@ sendBegin unitTests fuzzTests maybeReport = ) -sendResult : Int -> String -> List String -> String -> Decode.Value -> Decode.Value -> Cmd msg +sendResult : Int -> String -> List String -> Maybe String -> Decode.Value -> Decode.Value -> Cmd msg sendResult testId jsDefinitionName labels expectationElmCode debugLogs report = elmTestPort__send (Encode.object @@ -43,7 +43,7 @@ sendResult testId jsDefinitionName labels expectationElmCode debugLogs report = , ( "testId", Encode.int testId ) , ( "jsDefinitionName", Encode.string jsDefinitionName ) , ( "labels", Encode.list Encode.string labels ) - , ( "expectationElmCode", Encode.string expectationElmCode ) + , ( "expectationElmCode", encodeMaybe Encode.string expectationElmCode ) , ( "debugLogs", debugLogs ) -- Test reporter specific: @@ -75,6 +75,16 @@ sendError message = ) +encodeMaybe : (a -> Encode.Value) -> Maybe a -> Encode.Value +encodeMaybe encoder maybe = + case maybe of + Just a -> + encoder a + + Nothing -> + Encode.null + + type JsMessage = RunUnitTest Int | RunFuzzTest Int @@ -90,11 +100,11 @@ decoder = decodeMessageFromType : String -> Decoder JsMessage decodeMessageFromType messageType = case messageType of - "RunUnitTest" -> + "UNIT" -> Decode.map RunUnitTest (Decode.field "testId" Decode.int) - "RunFuzzTest" -> + "FUZZ" -> Decode.map RunFuzzTest (Decode.field "testId" Decode.int) From 41a93cfb3fcae7bb63e70b4f8da07e089b744b51 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 01:57:42 +0200 Subject: [PATCH 41/81] foo -> checkTagged --- elm/src/Test/Runner/Node.elm | 8 ++++---- lib/Generate.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 2dbe1252..6fab7b45 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,4 @@ -module Test.Runner.Node exposing (foo, run, TestProgram, PreviousRun) +module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun) {-| @@ -8,7 +8,7 @@ module Test.Runner.Node exposing (foo, run, TestProgram, PreviousRun) Runs a test and outputs its results to the console. Exit code is 0 if tests passed and 2 if any failed. Returns 1 if something went wrong. -@docs foo, run, TestProgram, PreviousRun +@docs checkTagged, run, TestProgram, PreviousRun -} @@ -468,8 +468,8 @@ toIndexedDict list = |> Dict.fromList -foo : a -> JsDefinitionName -> Maybe Test -foo value jsDefinitionName = +checkTagged : a -> JsDefinitionName -> Maybe Test +checkTagged value jsDefinitionName = check value |> Maybe.map (Test.Runner.tagTest jsDefinitionName) diff --git a/lib/Generate.js b/lib/Generate.js index 69922294..8449f388 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -325,7 +325,7 @@ main = function makeModuleTuple(mod) { const list = mod.possiblyTests.map((test) => { const name = toCompiledJavaScriptName(mod.moduleName, test); - return `Test.Runner.Node.foo ${mod.moduleName}.${test} "${name}"`; + return `Test.Runner.Node.checkTagged ${mod.moduleName}.${test} "${name}"`; }); return ` From ff00185825844a68ce73e01a6bb14c62f2aa74b0 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 03:24:20 +0200 Subject: [PATCH 42/81] Supervisor compiles --- elm/src/Test/Runner/Node.elm | 14 +- elm/src/Test/Runner/Ports.elm | 13 +- lib/Generate.js | 76 ++++++--- lib/RunTests.js | 3 +- lib/Supervisor.js | 308 +++++++++++++++++++++------------- 5 files changed, 259 insertions(+), 155 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 6fab7b45..ddfbb780 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -46,7 +46,6 @@ type alias DebugLogs = type alias InitArgs = { initialSeed : Int - , processes : Int , globs : List String , paths : List String , fuzzRuns : Int @@ -62,7 +61,6 @@ type alias RunnerOptions = , report : Report , globs : List String , paths : List String - , processes : Int , previousRun : PreviousRun } @@ -72,7 +70,6 @@ type alias Model = , fuzzTests : Dict TestId FuzzTest , runInfo : RunInfo , testReporter : TestReporter - , processes : Int , autoFail : Maybe String , previousRun : PreviousRun } @@ -193,7 +190,7 @@ dispatchUnitTest testId model = Just (Debug.toString expectation) in ( model - , Ports.sendResult testId jsDefinitionName unitTest.labels expectationElmCode debugLogs report + , Ports.sendResult testId False jsDefinitionName unitTest.labels expectationElmCode debugLogs report ) @@ -327,7 +324,7 @@ dispatchFuzzTest testId model = |> Just in ( model - , Ports.sendResult testId jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report + , Ports.sendResult testId True jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report ) @@ -378,7 +375,7 @@ update msg ({ testReporter } as model) = init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } index = +init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } index = let autoFail = case ( tests.seenOnly, tests.seenSkip ) of @@ -411,7 +408,6 @@ init { processes, globs, paths, fuzzRuns, initialSeed, report, tests, previousRu , fuzzRuns = fuzzRuns , initialSeed = initialSeed } - , processes = processes , testReporter = testReporter , autoFail = autoFail , previousRun = previousRun @@ -444,7 +440,6 @@ failInit message report _ = , fuzzRuns = 0 , initialSeed = 0 } - , processes = 0 , testReporter = createReporter report , autoFail = Nothing , previousRun = @@ -521,7 +516,7 @@ placeholderReplaceMe___ name = {-| Run the tests. -} run : RunnerOptions -> List ( String, List (Maybe Test) ) -> Program Int Model Msg -run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = +run { runs, seed, report, globs, paths, previousRun } possiblyTests = let testsList = possiblyTests @@ -553,7 +548,6 @@ run { runs, seed, report, globs, paths, processes, previousRun } possiblyTests = wrappedInit = init { initialSeed = seed - , processes = processes , globs = globs , paths = paths , fuzzRuns = runs diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm index 4286a4dd..f927a005 100644 --- a/elm/src/Test/Runner/Ports.elm +++ b/elm/src/Test/Runner/Ports.elm @@ -35,12 +35,21 @@ sendBegin unitTests fuzzTests maybeReport = ) -sendResult : Int -> String -> List String -> Maybe String -> Decode.Value -> Decode.Value -> Cmd msg -sendResult testId jsDefinitionName labels expectationElmCode debugLogs report = +sendResult : Int -> Bool -> String -> List String -> Maybe String -> Decode.Value -> Decode.Value -> Cmd msg +sendResult testId isFuzzTest jsDefinitionName labels expectationElmCode debugLogs report = elmTestPort__send (Encode.object [ ( "type", Encode.string "RESULT" ) , ( "testId", Encode.int testId ) + , ( "testType" + , Encode.string + (if isFuzzTest then + "fuzz" + + else + "unit" + ) + ) , ( "jsDefinitionName", Encode.string jsDefinitionName ) , ( "labels", Encode.list Encode.string labels ) , ( "expectationElmCode", encodeMaybe Encode.string expectationElmCode ) diff --git a/lib/Generate.js b/lib/Generate.js index 8449f388..1f446fac 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -259,7 +259,6 @@ function getModule(generatedCodeDir, moduleName) { possiblyTests: Array, }> } testModules * @param { Module } mainModule - * @param { number } processes * @returns { void } */ function generateMainModule( @@ -269,12 +268,11 @@ function generateMainModule( testFileGlobs, testFilePaths, testModules, - mainModule, - processes + mainModule ) { const testFileBody = makeTestFileBody( testModules, - makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths, processes) + makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) ); const testFileContents = `module ${mainModule.moduleName} exposing (main)\n\n${testFileBody}`; @@ -373,22 +371,13 @@ function indentAllButFirstLine(indent, string) { * @param { import('./Report').Report } report * @param { Array } testFileGlobs * @param { Array } testFilePaths - * @param { number } processes * @returns { string } */ -function makeOptsCode( - fuzz, - seed, - report, - testFileGlobs, - testFilePaths, - processes -) { +function makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) { return ` { runs = ${fuzz} , report = ${generateElmReportVariant(report)} , seed = ${seed} -, processes = ${processes} , previousRun = ${previousRunModuleName.join('.')}.previousRun , globs = ${indentAllButFirstLine(' ', makeList(testFileGlobs.map(makeElmString)))} @@ -439,34 +428,69 @@ function ensurePreviousRunModule(previousRunModule) { generatePreviousRunModule(previousRunModule, { fuzzRuns: -1, initialSeed: -1, - fingerprints: {}, + cachedTests: {}, }); } /** + * TODO: Sync with Elm. * @typedef { { fuzzRuns: number, initialSeed: number, - fingerprints: Record }> + cachedTests: Record } } PreviousRun * + * @typedef { { + hash: string, + unitTests: Array<{ labels: Array, expectation: string, debugLogs: Array }>, + fuzzTests: Array<{ labels: Array, expectation: string, debugLogs: Array }>, + } } CachedTests + * * @param { Module } previousRunModule * @param { PreviousRun } previousRun * @returns { void } */ function generatePreviousRunModule(previousRunModule, previousRun) { - const fingerprintsList = makeList( - Object.entries(previousRun.fingerprints) - // If there are no outcomes, the value wasn’t a test. - // Remember that we collect _all_ exposed values and test them at runtime if they are tests or not. - .filter(([, { outcomes }]) => outcomes.length > 0) - .map(([jsIdentifierName, { hash, outcomes }]) => + /** + * @param { { labels: Array, expectation: string, debugLogs: Array } } data + * @returns + */ + const toTuple = ({ labels, expectation, debugLogs }) => + debugLogs.length === 0 + ? `( ${expectation}, [] )` + : ` +( ${indentAllButFirstLine(' ', makeList(labels.map(makeElmString)))} +, ( ${expectation} + , ${indentAllButFirstLine( + ' ', + makeList(debugLogs.map(makeElmString)) + )} + ) +) + `.trim(); + + const cachedTestsList = makeList( + Object.entries(previousRun.cachedTests) + .filter( + ([, { unitTests, fuzzTests }]) => + unitTests.length > 0 && fuzzTests.length > 0 + ) + .map(([jsIdentifierName, { hash, unitTests, fuzzTests }]) => ` ( ${makeElmString(jsIdentifierName)} , { hash = ${makeElmString(hash)} - , outcomes = + , unitTests = + Dict.fromList + ${indentAllButFirstLine( + ' ', + makeList(unitTests.map(toTuple)) + )} + , fuzzTests = Dict.fromList - ${indentAllButFirstLine(' ', makeList(outcomes))} + ${indentAllButFirstLine( + ' ', + makeList(fuzzTests.map(toTuple)) + )} } ) `.trim() @@ -486,9 +510,9 @@ previousRun : Test.Runner.Node.PreviousRun previousRun = { fuzzRuns = ${previousRun.fuzzRuns} , initialSeed = ${previousRun.initialSeed} - , fingerprints = + , cachedTests = Dict.fromList - ${indentAllButFirstLine(' ', fingerprintsList)} + ${indentAllButFirstLine(' ', cachedTestsList)} } `.trim(); diff --git a/lib/RunTests.js b/lib/RunTests.js index 99410433..2a741b17 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -260,8 +260,7 @@ function runTests( testFileGlobs, testFilePaths, testModules, - mainModule, - processes + mainModule ); Generate.ensurePreviousRunModule(previousRunModule); diff --git a/lib/Supervisor.js b/lib/Supervisor.js index d6b88e1b..11d8a400 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -33,28 +33,32 @@ function run( watch ) { return new Promise(function (resolve) { - /** @type { number | null } */ - var nextResultToPrint = null; - var finishedWorkers = 0; + var unitTests = 0; + var fuzzTests = 0; + var finishedUnitTests = 0; + var finishedFuzzTests = 0; + var initializedWorkers = 0; var closedWorkers = 0; var results = new Map(); var failures = 0; /** @type { Array<{ labels: Array, todo: string }> } */ var todos = []; - var testsToRun = -1; var startingTime = Date.now(); /** @type { Array } */ var workers = []; + /** @type { import('net').Server | undefined } */ + var server = undefined; /** @type { import('./Generate').PreviousRun } */ var toBePreviousRun = { fuzzRuns: fuzz, initialSeed: seed, - fingerprints: {}, + cachedTests: {}, }; for (var key in hashes) { - toBePreviousRun.fingerprints[key] = { + toBePreviousRun.cachedTests[key] = { hash: hashes[key], - outcomes: [], + unitTests: [], + fuzzTests: [], }; } @@ -116,24 +120,6 @@ function run( } } - function flushResults() { - // Only print any results if we're ready - that is, nextResultToPrint - // is no longer null. (BEGIN changes it from null to 0.) - if (nextResultToPrint !== null) { - var result = results.get(nextResultToPrint); - - while ( - // If there are no more results to print, then we're done. - nextResultToPrint < testsToRun && - // Otherwise, keep going until we have no result available to print. - typeof result !== 'undefined' - ) { - printResult(result); - nextResultToPrint++; - result = results.get(nextResultToPrint); - } - } - } function reportRuntimeException() { console.error( chalk.red( @@ -143,10 +129,11 @@ function run( } /** - * @param { any } response This `any` became explicit instead of implicit when migrating from Flow to TypeScript. + * @param { number } testId + * @param { any } result This `any` became explicit instead of implicit when migrating from Flow to TypeScript. * @returns { void } */ - function handleResults(response) { + function handleResult(testId, result) { // TODO print progress bar - e.g. "Running test 5 of 20" on a bar! // -- yikes, be careful though...test the scenario where test // authors put Debug.log in their tests - does that mess @@ -156,51 +143,39 @@ function run( // backtrack the line feed, so that if someone else does more // logging, it will overwrite our status update and that's ok? - Object.keys(response.results).forEach(function (index) { - var result = response.results[index]; - results.set(parseInt(index), result); + if (report === 'junit') { + results.set(testId, result); + } - switch (report) { - case 'console': - switch (result.status) { - case 'pass': - // It's a PASS; no need to take any action. - break; - case 'todo': - todos.push(result); - break; - case 'fail': - failures++; - break; - default: - throw new Error(`Unexpected result.status: ${result.status}`); - } - break; - case 'junit': - if (typeof result.failure !== 'undefined') { - failures++; - } - break; - case 'json': - if (result.status === 'fail') { + switch (report) { + case 'console': + switch (result.status) { + case 'pass': + // It's a PASS; no need to take any action. + break; + case 'todo': + todos.push(result); + break; + case 'fail': failures++; - } else if (result.status === 'todo') { - todos.push({ labels: result.labels, todo: result.failures[0] }); - } - break; - } - }); - - for (const newStuff of response.newStuff) { - // Ignore bare `Test.todo`, which use the empty string – see Node.elm. - if (newStuff.jsDefinitionName !== '') { - toBePreviousRun.fingerprints[newStuff.jsDefinitionName].outcomes.push( - newStuff.dictTupleElmCode - ); - } + break; + default: + throw new Error(`Unexpected result.status: ${result.status}`); + } + break; + case 'junit': + if (typeof result.failure !== 'undefined') { + failures++; + } + break; + case 'json': + if (result.status === 'fail') { + failures++; + } else if (result.status === 'todo') { + todos.push({ labels: result.labels, todo: result.failures[0] }); + } + break; } - - flushResults(); } /** @@ -224,34 +199,51 @@ function run( socket.write(JSON.stringify(message)); }); }); + + socket.write( + JSON.stringify({ + type: 'FUZZ', + testId: initializedWorkers++, + }) + ); } /** - * @param { any } response This `any` became explicit instead of implicit when extracting this function. + * @typedef { + | { + type: 'BEGIN', + unitTests: number, + fuzzTests: number, + message?: any, + } + | { + type: 'RESULT', + testId: number, + testType: 'unit' | 'fuzz', + jsDefinitionName: string, + labels: Array, + expectationElmCode: string | null, + debugLogs: Array, + message: any, + } + | { + type: 'SUMMARY', + exitCode: number, + message: any, + } + | { + type: 'ERROR', + message: string, + } + } RunnerMessage - Needs to be in sync with Ports.elm. + * + * @param { RunnerMessage } response * @param { (message: any) => void } send * @returns { void } */ function handleResponse(response, send) { switch (response.type) { - case 'FINISHED': - handleResults(response); - - // This worker found no tests remaining to run; it's finished! - finishedWorkers++; - - // If all the workers have finished (or we run single-threaded), print the summary. - if (finishedWorkers === workers.length || processes === 1) { - send({ - type: 'SUMMARY', - duration: Date.now() - startingTime, - failures: failures, - todos: todos, - }); - } - break; case 'SUMMARY': - flushResults(); - if (response.exitCode === 1) { // The tests could not even run. At the time of this writing, the // only case is “No exposed values of type Test found”. That @@ -271,8 +263,8 @@ function run( } Generate.generatePreviousRunModule( - previousRunModule, - toBePreviousRun + previousRunModule, + toBePreviousRun ); } @@ -282,8 +274,10 @@ function run( }); end(response.exitCode); break; + case 'BEGIN': - testsToRun = response.testCount; + unitTests = response.unitTests; + fuzzTests = response.fuzzTests; if (!Report.isMachineReadable(report)) { var headline = 'elm-test ' + elmTestVersion; @@ -294,51 +288,135 @@ function run( printResult(response.message); - // Now we're ready to print results! - nextResultToPrint = 0; - - flushResults(); + // If running multi-threaded, run fuzz tests on threads. + // Save one core for the main thread. + if (fuzzTests > 0 && processes > 1) { + startWorkers(Math.min(processes - 1, fuzzTests)); + } + // Run unit tests in the main thread. + sendToMainProcess({ + type: 'UNIT', + testId: 0, + }); break; - case 'RESULTS': - handleResults(response); + case 'RESULT': + handleResult(response.testId, response.message); + printResult(response.message); + if (response.debugLogs.length > 0) { + // TODO: Print labels if needed + for (const debugLog of response.debugLogs) { + console.error(debugLog); + } + } + + if (response.expectationElmCode !== null) { + const cachedTests = + toBePreviousRun.cachedTests[response.jsDefinitionName]; + switch (response.testType) { + case 'unit': + cachedTests.unitTests.push({ + labels: response.labels, + expectation: response.expectationElmCode, + debugLogs: response.debugLogs, + }); + break; + + case 'fuzz': + cachedTests.fuzzTests.push({ + labels: response.labels, + expectation: response.expectationElmCode, + debugLogs: response.debugLogs, + }); + break; + } + } + + switch (response.testType) { + case 'unit': + finishedUnitTests++; + if (finishedUnitTests < unitTests) { + send({ + type: 'UNIT', + testId: finishedUnitTests, + }); + } else if (processes === 1 && finishedFuzzTests < fuzzTests) { + send({ + type: 'FUZZ', + testId: finishedFuzzTests, + }); + } + break; + + case 'fuzz': + finishedFuzzTests++; + if (finishedFuzzTests < fuzzTests) { + send({ + type: 'FUZZ', + testId: finishedFuzzTests, + }); + } + break; + } + + if ( + finishedUnitTests >= unitTests && + finishedFuzzTests >= fuzzTests + ) { + sendToMainProcess({ + type: 'SUMMARY', + duration: Date.now() - startingTime, + failures: failures, + todos: todos, + }); + } break; + case 'ERROR': throw new Error(response.message); + default: - throw new Error('Unrecognized message from worker:' + response.type); + throw new Error( + 'Unrecognized message from worker: ' + + /** @type { { type: string } } */ (response).type + ); } } - // If just one process, run single-threaded. - if (processes === 1) { - var { run } = require(dest); - // Allow the generated file to be `require`d again (for watch mode). - delete require.cache[dest]; - var send = run( - 0, - /** @type { (response: any) => void } */ - (response) => { - handleResponse(response, send); - } - ); - } else { + var sendToMainProcess = require(dest).run( + 0, + /** @type { (response: any) => void } */ + (response) => { + handleResponse(response, sendToMainProcess); + } + ); + + // Allow the generated file to be `require`d again (for watch mode). + delete require.cache[dest]; + + /** + * @param { number } amount + * @returns { void } + */ + function startWorkers(amount) { var pendingException = false; // Using a named pipe to communicate is actually faster than // using `process.send` or `worker_threads`! See: // https://github.com/rtfeldman/node-test-runner/pull/674 - var server = net.createServer(initWorker); + server = net.createServer(initWorker); server.on('error', function (err) { console.error(err.stack); - server.close(); + if (server) { + server.close(); + } }); server.on('listening', function () { - workers = Array.from({ length: processes }, (_, index) => { - var worker = child_process.fork(dest, [index.toString()]); + workers = Array.from({ length: amount }, (_, index) => { + var worker = child_process.fork(dest, [(index + 1).toString()]); worker.on('close', function (code) { // code can be null. From fbbd20480b4ac8b131bae059ebb6be3a8f477bea Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 03:40:02 +0200 Subject: [PATCH 43/81] Print which test the debug logs are from --- elm/src/Test/Reporter/Console.elm | 14 +++++++--- elm/src/Test/Reporter/JUnit.elm | 1 + elm/src/Test/Reporter/TestResults.elm | 38 +-------------------------- elm/src/Test/Runner/Node.elm | 14 +++++++--- lib/Supervisor.js | 10 ++++--- 5 files changed, 30 insertions(+), 47 deletions(-) diff --git a/elm/src/Test/Reporter/Console.elm b/elm/src/Test/Reporter/Console.elm index ad8aed30..26ab5a34 100644 --- a/elm/src/Test/Reporter/Console.elm +++ b/elm/src/Test/Reporter/Console.elm @@ -147,7 +147,7 @@ getStatus outcome = reportComplete : UseColor -> Results.TestResult -> Value -reportComplete useColor { labels, outcome } = +reportComplete useColor { labels, outcome, hasDebugLogs } = Encode.object <| ( "type", Encode.string "complete" ) :: ( "status", Encode.string (getStatus outcome) ) @@ -156,10 +156,18 @@ reportComplete useColor { labels, outcome } = -- No failures of any kind. case distributionReportToString distributionReport of Nothing -> - [] + if hasDebugLogs then + [ ( "message" + , passedLabelsToText labels + |> textToValue useColor + ) + ] + + else + [] Just report -> - [ ( "distributionReport" + [ ( "message" , report |> passedToText labels |> textToValue useColor diff --git a/elm/src/Test/Reporter/JUnit.elm b/elm/src/Test/Reporter/JUnit.elm index 689d58c2..61e20f9f 100644 --- a/elm/src/Test/Reporter/JUnit.elm +++ b/elm/src/Test/Reporter/JUnit.elm @@ -119,6 +119,7 @@ encodeExtraFailure _ = } , NoDistribution ) + , hasDebugLogs = False } diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index 5380d4bf..68047851 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -3,8 +3,6 @@ module Test.Reporter.TestResults exposing , Outcome(..) , SummaryInfo , TestResult - , isFailure - , outcomeFromExpectations ) import Expect exposing (Expectation) @@ -23,6 +21,7 @@ type alias TestResult = { labels : List String , outcome : Outcome , duration : Float -- in milliseconds + , hasDebugLogs : Bool } @@ -40,38 +39,3 @@ type alias Failure = , description : String , reason : Reason } - - -isFailure : Outcome -> Bool -isFailure outcome = - case outcome of - Failed _ -> - True - - _ -> - False - - -outcomeFromExpectations : List Expectation -> Outcome -outcomeFromExpectations expectations = - case expectations of - -- The type of test runner functions says that they return `List Expectation`, - -- but in practice they only ever return lists with exactly one item: - -- https://github.com/elm-explorations/test/pull/244 - -- That PR was reverted because it unfortunately was a breaking change for the package: - -- https://github.com/elm-explorations/test/commit/11f70d5fc0b6fdc88d7a34ea1d10f56969890493 - -- But to keep things simpler here, we only support exactly one expectation. - [ expectation ] -> - case Test.Runner.getFailureReason expectation of - Nothing -> - Passed (Test.Runner.getDistributionReport expectation) - - Just failure -> - if Test.Runner.isTodo expectation then - Todo failure.description - - else - Failed ( failure, Test.Runner.getDistributionReport expectation ) - - _ -> - Debug.todo ("A test somehow did not return exactly 1 expectation, it returned " ++ String.fromInt (List.length expectations) ++ "!") diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index ddfbb780..b66d1609 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -22,7 +22,7 @@ import Task import Test exposing (Test) import Test.Distribution exposing (DistributionReport(..)) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) -import Test.Reporter.TestResults exposing (Outcome(..), TestResult, isFailure, outcomeFromExpectations) +import Test.Reporter.TestResults exposing (Outcome(..), TestResult) import Test.Runner exposing (FuzzTest, FuzzTestExpectation(..), Tests, UnitTest, UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..)) import Test.Runner.Ports as Ports exposing (JsMessage(..)) @@ -154,6 +154,9 @@ dispatchUnitTest testId model = Nothing -> ( runWithDuration unitTest.thunk, getAndClearDebugLogs False ) + hasDebugLogs = + isEmptyDebugLogs debugLogs + outcome = case expectation of UnitTestPass -> @@ -177,13 +180,14 @@ dispatchUnitTest testId model = { labels = unitTest.labels , outcome = outcome , duration = duration + , hasDebugLogs = hasDebugLogs } report = model.testReporter.reportComplete result expectationElmCode = - if expectation == UnitTestPass && isEmptyDebugLogs debugLogs then + if expectation == UnitTestPass && hasDebugLogs then Nothing else @@ -289,6 +293,9 @@ dispatchFuzzTest testId model = ) ) + hasDebugLogs = + isEmptyDebugLogs debugLogs + outcome = case expectation of FuzzTestPass { distributionReport } -> @@ -308,13 +315,14 @@ dispatchFuzzTest testId model = { labels = fuzzTest.labels , outcome = outcome , duration = duration + , hasDebugLogs = hasDebugLogs } report = model.testReporter.reportComplete result expectationElmCode = - if expectation == FuzzTestPass { distributionReport = NoDistribution } && isEmptyDebugLogs debugLogs then + if expectation == FuzzTestPass { distributionReport = NoDistribution } && hasDebugLogs then Nothing else diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 11d8a400..9cfda357 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -87,9 +87,9 @@ function run( case 'complete': switch (result.status) { case 'pass': - // passed tests should be printed only if they contain distributionReport - if (result.distributionReport !== undefined) { - console.log(makeWindowsSafe(result.distributionReport)); + // passed tests should be printed only if they contain debug logs or a distributionReport + if (result.message !== undefined) { + console.log(makeWindowsSafe(result.message)); } break; case 'todo': @@ -305,7 +305,9 @@ function run( handleResult(response.testId, response.message); printResult(response.message); if (response.debugLogs.length > 0) { - // TODO: Print labels if needed + if (report !== 'console') { + console.error(response.labels.slice().reverse().join(' > ')); + } for (const debugLog of response.debugLogs) { console.error(debugLog); } From 6abe2b2db1882fb3551e5d84da4c1836bac261eb Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 03:47:55 +0200 Subject: [PATCH 44/81] Typed sends as well --- lib/Supervisor.js | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 9cfda357..292317fb 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -194,18 +194,19 @@ function run( crlfDelay: Infinity, }); + /** @type { SendToWorker } */ + const send = (message) => { + socket.write(JSON.stringify(message)); + }; + stream.on('line', function (data) { - handleResponse(JSON.parse(data), (message) => { - socket.write(JSON.stringify(message)); - }); + handleResponse(JSON.parse(data), send); }); - socket.write( - JSON.stringify({ - type: 'FUZZ', - testId: initializedWorkers++, - }) - ); + send({ + type: 'FUZZ', + testId: initializedWorkers++, + }); } /** @@ -235,10 +236,28 @@ function run( type: 'ERROR', message: string, } - } RunnerMessage - Needs to be in sync with Ports.elm. + } FromWorkerMessage - Needs to be in sync with Ports.elm. + * + * @typedef { (message: ToWorkerMessage) => void } SendToWorker + * @typedef { + | { + type: 'UNIT', + testId: number, + } + | { + type: 'FUZZ', + testId: number, + } + | { + type: 'SUMMARY', + duration: number, + failures: number, + todos: Array<{ labels: Array, todo: string }>, + } + } ToWorkerMessage - Needs to be in sync with Ports.elm. * - * @param { RunnerMessage } response - * @param { (message: any) => void } send + * @param { FromWorkerMessage } response + * @param { SendToWorker } send * @returns { void } */ function handleResponse(response, send) { @@ -386,9 +405,10 @@ function run( } } + /** @type { SendToWorker } */ var sendToMainProcess = require(dest).run( 0, - /** @type { (response: any) => void } */ + /** @type { (response: FromWorkerMessage) => void } */ (response) => { handleResponse(response, sendToMainProcess); } From 0ce6e026e75998159b093d6e003d87fcbf287cb6 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 03:55:06 +0200 Subject: [PATCH 45/81] Fix mistakes in code generation --- lib/Generate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Generate.js b/lib/Generate.js index 1f446fac..53d54b03 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -83,7 +83,7 @@ const getHashDefinition = placeholderDefinition('getHash'); const runWithDurationDefinition = placeholderDefinition('runWithDuration'); const debugLogDefinition = - /^var _Debug_log = F2\(function \(tag, value\)\s*\{[^}]+\}\)/m; + /^var _Debug_log = F2\(function\(tag, value\)\s*\{[^}]+\}\)/m; /** * @param { string } moduleName @@ -138,7 +138,7 @@ var _Debug_log = F2(function(tag, value) ) .replace( getAndClearDebugLogsDefinition, - '$1 = pause => { var logs = _Json_wrap(_Debug_logs); _Debug_logs = []; _Debug_logPaused = paused; return logs; }' + '$1 = paused => { var logs = _Json_wrap(_Debug_logs); _Debug_logs = []; _Debug_logPaused = paused; return logs; }' ) .replace( getHashDefinition, From e27c31a584e244f955a9bc5f4f29f65082be0e63 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 03:57:39 +0200 Subject: [PATCH 46/81] Fix flipped hasDebugLogs --- elm/src/Test/Runner/Node.elm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index b66d1609..2b62c947 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -155,7 +155,7 @@ dispatchUnitTest testId model = ( runWithDuration unitTest.thunk, getAndClearDebugLogs False ) hasDebugLogs = - isEmptyDebugLogs debugLogs + not (isEmptyDebugLogs debugLogs) outcome = case expectation of @@ -187,7 +187,7 @@ dispatchUnitTest testId model = model.testReporter.reportComplete result expectationElmCode = - if expectation == UnitTestPass && hasDebugLogs then + if expectation == UnitTestPass && not hasDebugLogs then Nothing else @@ -294,7 +294,7 @@ dispatchFuzzTest testId model = ) hasDebugLogs = - isEmptyDebugLogs debugLogs + not (isEmptyDebugLogs debugLogs) outcome = case expectation of @@ -322,7 +322,7 @@ dispatchFuzzTest testId model = model.testReporter.reportComplete result expectationElmCode = - if expectation == FuzzTestPass { distributionReport = NoDistribution } && hasDebugLogs then + if expectation == FuzzTestPass { distributionReport = NoDistribution } && not hasDebugLogs then Nothing else From e3c6ce5cc9966c8ca3f5c14540439ca57fa2478d Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:03:54 +0200 Subject: [PATCH 47/81] TODO comment about Debug.todo --- lib/Generate.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Generate.js b/lib/Generate.js index 53d54b03..f7458a5b 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -122,6 +122,7 @@ function patch(hashes, content) { ` var _Debug_logs = []; var _Debug_logPaused = false; +// TODO: Debug.todo does not work like I thought it would. Not a good tip as is. var _Debug_logPausedMessage = 'For passing fuzz tests, Debug.log is not shown, since showing logs from lots of runs is pretty confusing. Tip: Use Debug.todo to fail a test from anywhere.'; var _Debug_log = F2(function(tag, value) { From 7b847611dc41652db1b5086f4a8e2042a8f4b645 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:04:11 +0200 Subject: [PATCH 48/81] Make room after debug logs --- lib/Supervisor.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 292317fb..ecfcbb58 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -330,6 +330,9 @@ function run( for (const debugLog of response.debugLogs) { console.error(debugLog); } + if (report === 'console') { + console.error('\n'); + } } if (response.expectationElmCode !== null) { From e7af247badae75c8f5ddf2ba670e6ec0ec0b2dcd Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:14:48 +0200 Subject: [PATCH 49/81] Reorder since send is sync on main thread --- lib/Supervisor.js | 74 +++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/lib/Supervisor.js b/lib/Supervisor.js index ecfcbb58..62bb2321 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -320,7 +320,7 @@ function run( }); break; - case 'RESULT': + case 'RESULT': { handleResult(response.testId, response.message); printResult(response.message); if (response.debugLogs.length > 0) { @@ -335,50 +335,27 @@ function run( } } - if (response.expectationElmCode !== null) { - const cachedTests = - toBePreviousRun.cachedTests[response.jsDefinitionName]; - switch (response.testType) { - case 'unit': + const cachedTests = + toBePreviousRun.cachedTests[response.jsDefinitionName]; + switch (response.testType) { + case 'unit': + finishedUnitTests++; + if (response.expectationElmCode !== null) { cachedTests.unitTests.push({ labels: response.labels, expectation: response.expectationElmCode, debugLogs: response.debugLogs, }); - break; - - case 'fuzz': - cachedTests.fuzzTests.push({ - labels: response.labels, - expectation: response.expectationElmCode, - debugLogs: response.debugLogs, - }); - break; - } - } - - switch (response.testType) { - case 'unit': - finishedUnitTests++; - if (finishedUnitTests < unitTests) { - send({ - type: 'UNIT', - testId: finishedUnitTests, - }); - } else if (processes === 1 && finishedFuzzTests < fuzzTests) { - send({ - type: 'FUZZ', - testId: finishedFuzzTests, - }); } break; case 'fuzz': finishedFuzzTests++; - if (finishedFuzzTests < fuzzTests) { - send({ - type: 'FUZZ', - testId: finishedFuzzTests, + if (response.expectationElmCode !== null) { + cachedTests.fuzzTests.push({ + labels: response.labels, + expectation: response.expectationElmCode, + debugLogs: response.debugLogs, }); } break; @@ -394,8 +371,35 @@ function run( failures: failures, todos: todos, }); + } else { + switch (response.testType) { + case 'unit': + if (finishedUnitTests < unitTests) { + send({ + type: 'UNIT', + testId: finishedUnitTests, + }); + } else if (processes === 1 && finishedFuzzTests < fuzzTests) { + send({ + type: 'FUZZ', + testId: finishedFuzzTests, + }); + } + break; + + case 'fuzz': + if (finishedFuzzTests < fuzzTests) { + send({ + type: 'FUZZ', + testId: finishedFuzzTests, + }); + } + break; + } } + break; + } case 'ERROR': throw new Error(response.message); From 56355a0f27be364ae5e398dfd60debbad791a8ff Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:31:12 +0200 Subject: [PATCH 50/81] Keep track of next and finished separately --- lib/Supervisor.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 62bb2321..d6ecbdd9 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -35,9 +35,10 @@ function run( return new Promise(function (resolve) { var unitTests = 0; var fuzzTests = 0; + var nextUnitTest = 0; + var nextFuzzTest = 0; var finishedUnitTests = 0; var finishedFuzzTests = 0; - var initializedWorkers = 0; var closedWorkers = 0; var results = new Map(); var failures = 0; @@ -205,7 +206,7 @@ function run( send({ type: 'FUZZ', - testId: initializedWorkers++, + testId: nextFuzzTest++, }); } @@ -316,7 +317,7 @@ function run( // Run unit tests in the main thread. sendToMainProcess({ type: 'UNIT', - testId: 0, + testId: nextUnitTest++, }); break; @@ -374,24 +375,24 @@ function run( } else { switch (response.testType) { case 'unit': - if (finishedUnitTests < unitTests) { + if (nextUnitTest < unitTests) { send({ type: 'UNIT', - testId: finishedUnitTests, + testId: nextUnitTest++, }); - } else if (processes === 1 && finishedFuzzTests < fuzzTests) { + } else if (processes === 1 && nextFuzzTest < fuzzTests) { send({ type: 'FUZZ', - testId: finishedFuzzTests, + testId: nextFuzzTest++, }); } break; case 'fuzz': - if (finishedFuzzTests < fuzzTests) { + if (nextFuzzTest < fuzzTests) { send({ type: 'FUZZ', - testId: finishedFuzzTests, + testId: nextFuzzTest++, }); } break; From 4e296b782b5267f38ede97939e5a5071b38d4573 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:32:21 +0200 Subject: [PATCH 51/81] Explain rounding --- elm/src/Test/Reporter/Json.elm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elm/src/Test/Reporter/Json.elm b/elm/src/Test/Reporter/Json.elm index 06b8d5ac..165f2cd5 100644 --- a/elm/src/Test/Reporter/Json.elm +++ b/elm/src/Test/Reporter/Json.elm @@ -28,6 +28,8 @@ reportComplete { duration, labels, outcome } = , ( "labels", encodeLabels labels ) , ( "failures", Encode.list identity (encodeFailures outcome) ) , ( "distributionReports", Encode.list identity (encodeDistributionReports outcome) ) + + -- We have a more exact float these days, but to avoid a breaking change we round it to an int. , ( "duration", Encode.string <| String.fromInt (round duration) ) ] From 826855c17d6c29ce0a8dca7b9cac6061aadec1fc Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:36:14 +0200 Subject: [PATCH 52/81] Comments cleanup --- lib/Generate.js | 1 - lib/Tarjan.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Generate.js b/lib/Generate.js index f7458a5b..97744e81 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -434,7 +434,6 @@ function ensurePreviousRunModule(previousRunModule) { } /** - * TODO: Sync with Elm. * @typedef { { fuzzRuns: number, initialSeed: number, diff --git a/lib/Tarjan.js b/lib/Tarjan.js index ba19fc4a..81c1cd2d 100644 --- a/lib/Tarjan.js +++ b/lib/Tarjan.js @@ -23,7 +23,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI get: (key: string) => Set, } } Graph * - * @param { Graph } graph + * @param { Graph } graph * @returns { Array> } */ function stronglyConnectedComponents(graph) { From eca89d8afcc6c7aaf033517f8384cc7048e56ebb Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:46:07 +0200 Subject: [PATCH 53/81] PreviousRun fixes --- elm/src/Test/Runner/Node.elm | 20 ++++++++++---------- lib/Generate.js | 15 +++++---------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 2b62c947..d933ddfa 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -82,16 +82,6 @@ type alias PreviousRun = } -{-| A program which will run tests and report their results. --} -type alias TestProgram = - Platform.Program Int Model Msg - - -type Msg - = Receive (Result Decode.Error JsMessage) - - type alias CachedTests = { hash : String @@ -103,6 +93,16 @@ type alias CachedTests = } +{-| A program which will run tests and report their results. +-} +type alias TestProgram = + Platform.Program Int Model Msg + + +type Msg + = Receive (Result Decode.Error JsMessage) + + noDebugLogs : DebugLogs noDebugLogs = Encode.list never [] diff --git a/lib/Generate.js b/lib/Generate.js index 97744e81..7dc954ff 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -456,15 +456,10 @@ function generatePreviousRunModule(previousRunModule, previousRun) { * @returns */ const toTuple = ({ labels, expectation, debugLogs }) => - debugLogs.length === 0 - ? `( ${expectation}, [] )` - : ` -( ${indentAllButFirstLine(' ', makeList(labels.map(makeElmString)))} + ` +( ${indentAllButFirstLine(' ', makeList(labels.map(makeElmString)))} , ( ${expectation} - , ${indentAllButFirstLine( - ' ', - makeList(debugLogs.map(makeElmString)) - )} + , ${indentAllButFirstLine(' ', makeList(debugLogs.map(makeElmString)))} ) ) `.trim(); @@ -473,7 +468,7 @@ function generatePreviousRunModule(previousRunModule, previousRun) { Object.entries(previousRun.cachedTests) .filter( ([, { unitTests, fuzzTests }]) => - unitTests.length > 0 && fuzzTests.length > 0 + unitTests.length > 0 || fuzzTests.length > 0 ) .map(([jsIdentifierName, { hash, unitTests, fuzzTests }]) => ` @@ -502,7 +497,7 @@ module ${previousRunModule.moduleName} exposing (previousRun) import Dict import Test.Distribution exposing (DistributionReport(..)) -import Test.Reporter.TestResults exposing (Outcome(..)) +import Test.Runner exposing (FuzzTestExpectation(..), UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) import Test.Runner.Node From e74e8baf20a637b77fd64ac8beb3ffd7e96b220b Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 04:51:22 +0200 Subject: [PATCH 54/81] Fix debug logs compile error in cache file --- lib/Generate.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Generate.js b/lib/Generate.js index 7dc954ff..00431e3e 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -459,7 +459,8 @@ function generatePreviousRunModule(previousRunModule, previousRun) { ` ( ${indentAllButFirstLine(' ', makeList(labels.map(makeElmString)))} , ( ${expectation} - , ${indentAllButFirstLine(' ', makeList(debugLogs.map(makeElmString)))} + , Json.Encode.list Json.Encode.string + ${indentAllButFirstLine(' ', makeList(debugLogs.map(makeElmString)))} ) ) `.trim(); @@ -496,6 +497,7 @@ function generatePreviousRunModule(previousRunModule, previousRun) { module ${previousRunModule.moduleName} exposing (previousRun) import Dict +import Json.Encode import Test.Distribution exposing (DistributionReport(..)) import Test.Runner exposing (FuzzTestExpectation(..), UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) From 746e7cbffdf85c1ec3ccda4d40a468c497d8ae65 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 05:13:19 +0200 Subject: [PATCH 55/81] Optimize debug logs a bit --- lib/Generate.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/Generate.js b/lib/Generate.js index 00431e3e..4c4fe6c0 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -85,6 +85,12 @@ const runWithDurationDefinition = placeholderDefinition('runWithDuration'); const debugLogDefinition = /^var _Debug_log = F2\(function\(tag, value\)\s*\{[^}]+\}\)/m; +const encodeEmptyDebugLogsCall = + /^(\s*)\$author\$project\$Test\$Generated\$PreviousRun\$encodeDebugLogs\(_List_Nil\)/gm; + +const encodeDebugLogsCall = + /^(\s*)\$author\$project\$Test\$Generated\$PreviousRun\$encodeDebugLogs\(\s*_List_fromArray\(/gm; + /** * @param { string } moduleName * @param { string } valueName @@ -153,6 +159,10 @@ var _Debug_log = F2(function(tag, value) runWithDurationDefinition, '$1 = thunk => { var t = performance.now(); return _Utils_Tuple2(thunk(null), performance.now() - t); }' ) + // This optimizes a tiny bit: Instead of having a JS array, turning it into an Elm list, + // and then encoding it back to a JS array again, we just wrap the original array. + .replace(encodeEmptyDebugLogsCall, '$1_Json_wrap([])') + .replace(encodeDebugLogsCall, '$1(_Json_wrap(') ); } @@ -459,7 +469,7 @@ function generatePreviousRunModule(previousRunModule, previousRun) { ` ( ${indentAllButFirstLine(' ', makeList(labels.map(makeElmString)))} , ( ${expectation} - , Json.Encode.list Json.Encode.string + , encodeDebugLogs ${indentAllButFirstLine(' ', makeList(debugLogs.map(makeElmString)))} ) ) @@ -503,6 +513,12 @@ import Test.Runner exposing (FuzzTestExpectation(..), UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) import Test.Runner.Node + +encodeDebugLogs : List String -> Json.Encode.Value +encodeDebugLogs = + Json.Encode.list Json.Encode.string + + previousRun : Test.Runner.Node.PreviousRun previousRun = { fuzzRuns = ${previousRun.fuzzRuns} From 9db94d86610bb41602a9e80072b4ab489c45e127 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 10:25:46 +0200 Subject: [PATCH 56/81] index -> shouldSendBegin --- elm/src/Test/Runner/Node.elm | 13 ++++++------- example-application/tests/TestsPassing.elm | 1 + lib/Supervisor.js | 6 +++--- templates/after.js | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index d933ddfa..e854dd10 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -96,7 +96,7 @@ type alias CachedTests = {-| A program which will run tests and report their results. -} type alias TestProgram = - Platform.Program Int Model Msg + Program Bool Model Msg type Msg @@ -382,8 +382,8 @@ update msg ({ testReporter } as model) = ( model, Ports.sendError (Decode.errorToString err) ) -init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } index = +init : InitArgs -> Bool -> ( Model, Cmd Msg ) +init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldSendBegin = let autoFail = case ( tests.seenOnly, tests.seenSkip ) of @@ -422,8 +422,7 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } index = } in ( model - -- TODO: `index` doesn't really make sense anymore. - , if index == 0 then + , if shouldSendBegin then Ports.sendBegin (Dict.size model.unitTests) (Dict.size model.fuzzTests) @@ -434,7 +433,7 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } index = ) -failInit : String -> Report -> Int -> ( Model, Cmd Msg ) +failInit : String -> Report -> Bool -> ( Model, Cmd Msg ) failInit message report _ = let model : Model @@ -523,7 +522,7 @@ placeholderReplaceMe___ name = {-| Run the tests. -} -run : RunnerOptions -> List ( String, List (Maybe Test) ) -> Program Int Model Msg +run : RunnerOptions -> List ( String, List (Maybe Test) ) -> TestProgram run { runs, seed, report, globs, paths, previousRun } possiblyTests = let testsList = diff --git a/example-application/tests/TestsPassing.elm b/example-application/tests/TestsPassing.elm index 9500d5fd..f72205c6 100644 --- a/example-application/tests/TestsPassing.elm +++ b/example-application/tests/TestsPassing.elm @@ -10,6 +10,7 @@ testEqual = test "Expect.equal works" <| \() -> Something.ultimateAnswer + |> Debug.log "HÄR" |> Expect.equal 42 diff --git a/lib/Supervisor.js b/lib/Supervisor.js index d6ecbdd9..b72ebb02 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -415,7 +415,7 @@ function run( /** @type { SendToWorker } */ var sendToMainProcess = require(dest).run( - 0, + /* shouldSendBegin */ true, /** @type { (response: FromWorkerMessage) => void } */ (response) => { handleResponse(response, sendToMainProcess); @@ -445,8 +445,8 @@ function run( }); server.on('listening', function () { - workers = Array.from({ length: amount }, (_, index) => { - var worker = child_process.fork(dest, [(index + 1).toString()]); + workers = Array.from({ length: amount }, () => { + var worker = child_process.fork(dest); worker.on('close', function (code) { // code can be null. diff --git a/templates/after.js b/templates/after.js index 05898f84..182d40c6 100644 --- a/templates/after.js +++ b/templates/after.js @@ -1,5 +1,5 @@ -function run(index, receive) { - var app = Elm.Test.Generated.Main.init({ flags: index }); +function run(shouldSendBegin, receive) { + var app = Elm.Test.Generated.Main.init({ flags: shouldSendBegin }); // Without this, each run leaks memory in single-threaded mode: Elm = null; app.ports.elmTestPort__send.subscribe(receive); @@ -19,7 +19,7 @@ function main() { client.setEncoding('utf8'); client.setNoDelay(true); - var send = run(Number(process.argv[2]), function (msg) { + var send = run(false, function (msg) { // We split incoming messages on the socket on newlines. The gist is that node // is rather unpredictable in whether or not a single `write` will result in a // single `on('data')` callback. Sometimes it does, sometimes multiple writes From 6d32633f5055972a53923533bc7e0aa42366a318 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 11:30:38 +0200 Subject: [PATCH 57/81] Trawl cached at start --- elm/src/Test/Runner/Node.elm | 497 +++++++++++++++++++++------------- elm/src/Test/Runner/Ports.elm | 20 +- lib/Supervisor.js | 89 ++++-- 3 files changed, 389 insertions(+), 217 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index e854dd10..5358ae1e 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -72,6 +72,7 @@ type alias Model = , testReporter : TestReporter , autoFail : Maybe String , previousRun : PreviousRun + , cacheTrawl : CacheTrawl } @@ -93,6 +94,19 @@ type alias CachedTests = } +type CacheTrawl + = NotTrawling + | TrawlingUnitTests + { current : TestId + , unitTests : List TestId + } + | TrawlingFuzzTests + { current : TestId + , unitTests : List TestId + , fuzzTests : List TestId + } + + {-| A program which will run tests and report their results. -} type alias TestProgram = @@ -101,6 +115,7 @@ type alias TestProgram = type Msg = Receive (Result Decode.Error JsMessage) + | Trawl noDebugLogs : DebugLogs @@ -113,98 +128,75 @@ isEmptyDebugLogs debugLogs = Decode.decodeValue (Decode.field "length" Decode.int) debugLogs == Ok 0 -dispatchUnitTest : TestId -> Model -> ( Model, Cmd Msg ) +dispatchUnitTest : TestId -> Model -> Cmd Msg dispatchUnitTest testId model = case Dict.get testId model.unitTests of Nothing -> - ( model - , Ports.sendError ("Unit test not found: " ++ String.fromInt testId) - ) + Ports.sendError ("Unit test not found: " ++ String.fromInt testId) Just unitTest -> let - jsDefinitionName = - unitTest.tag - - hash = - getHash jsDefinitionName - - maybeCached = - Dict.get jsDefinitionName model.previousRun.cachedTests - |> Maybe.andThen - (\cachedTests -> - if hash == cachedTests.hash then - case Dict.get unitTest.labels cachedTests.unitTests of - -- As an optimization, passing unit tests without debug logs are not stored. - Nothing -> - Just ( UnitTestPass, noDebugLogs ) - - cached -> - cached - - else - Nothing - ) - + -- The unnecessary-looking tuple here ensures that `getAndClearDebugLogs` + -- runs _after_ the thunk. ( ( expectation, duration ), debugLogs ) = - case maybeCached of - Just ( expectation_, debugLogs_ ) -> - ( ( expectation_, 0 ), debugLogs_ ) + ( runWithDuration unitTest.thunk, getAndClearDebugLogs False ) + in + sendUnitTestResult testId unitTest expectation duration debugLogs model.testReporter - Nothing -> - ( runWithDuration unitTest.thunk, getAndClearDebugLogs False ) - hasDebugLogs = - not (isEmptyDebugLogs debugLogs) +sendUnitTestResult : TestId -> UnitTest -> UnitTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg +sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = + let + jsDefinitionName = + unitTest.tag - outcome = - case expectation of - UnitTestPass -> - Passed NoDistribution + hasDebugLogs = + not (isEmptyDebugLogs debugLogs) - UnitTestFail { description, reason } -> - if reason == TODO then - Todo description + outcome = + case expectation of + UnitTestPass -> + Passed NoDistribution - else - Failed - ( { given = Nothing - , description = description - , reason = reason - } - , NoDistribution - ) + UnitTestFail { description, reason } -> + if reason == TODO then + Todo description - result : TestResult - result = - { labels = unitTest.labels - , outcome = outcome - , duration = duration - , hasDebugLogs = hasDebugLogs - } + else + Failed + ( { given = Nothing + , description = description + , reason = reason + } + , NoDistribution + ) - report = - model.testReporter.reportComplete result + result : TestResult + result = + { labels = unitTest.labels + , outcome = outcome + , duration = duration + , hasDebugLogs = hasDebugLogs + } - expectationElmCode = - if expectation == UnitTestPass && not hasDebugLogs then - Nothing + report = + testReporter.reportComplete result - else - Just (Debug.toString expectation) - in - ( model - , Ports.sendResult testId False jsDefinitionName unitTest.labels expectationElmCode debugLogs report - ) + expectationElmCode = + if expectation == UnitTestPass && not hasDebugLogs then + Nothing + + else + Just (Debug.toString expectation) + in + Ports.sendResult testId False jsDefinitionName unitTest.labels expectationElmCode debugLogs report -dispatchFuzzTest : TestId -> Model -> ( Model, Cmd Msg ) +dispatchFuzzTest : TestId -> Model -> Cmd Msg dispatchFuzzTest testId model = case Dict.get testId model.fuzzTests of Nothing -> - ( model - , Ports.sendError ("Fuzz test not found: " ++ String.fromInt testId) - ) + Ports.sendError ("Fuzz test not found: " ++ String.fromInt testId) Just fuzzTest -> let @@ -214,126 +206,107 @@ dispatchFuzzTest testId model = hash = getHash jsDefinitionName - ( fuzzerInts, maybeCached ) = + fuzzerInts = case Dict.get jsDefinitionName model.previousRun.cachedTests of Nothing -> - ( [], Nothing ) + [] Just cachedTests -> - let - canUseCached = - (hash == cachedTests.hash) - && (model.runInfo.initialSeed == model.previousRun.initialSeed) - -- If the fuzz tests specifies its own number of runs and the hash is the same, - -- then the number of runs must be unchanged. - && (fuzzTest.runs /= Nothing || model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) - in case Dict.get fuzzTest.labels cachedTests.fuzzTests of - -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. Nothing -> - ( [] - , if canUseCached then - Just ( FuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) - - else - Nothing - ) - - (Just ( expectation_, debugLogs_ )) as cached -> - let - fuzzerInts_ = - case expectation_ of - FuzzTestPass _ -> - [] - - FuzzTestFail data -> - data.fuzzerInts - in - ( fuzzerInts_ - , if canUseCached then - cached - - else - Nothing - ) - - ( ( expectation, duration ), debugLogs ) = - case maybeCached of - Just ( expectation_, debugLogs_ ) -> - ( ( expectation_, 0 ), debugLogs_ ) - - Nothing -> - let - seed = - Random.initialSeed model.runInfo.initialSeed - in - getAndClearDebugLogs True - |> (\_ -> + [] + + Just ( expectation_, _ ) -> + case expectation_ of + FuzzTestPass _ -> + [] + + FuzzTestFail data -> + data.fuzzerInts + + seed = + Random.initialSeed model.runInfo.initialSeed + + ( expectation, duration, debugLogs ) = + -- Pause debug logs. + getAndClearDebugLogs True + |> (\_ -> + let + ( expectation_, duration_ ) = + runWithDuration (\() -> fuzzTest.thunk seed model.runInfo.fuzzRuns fuzzerInts) + in + case expectation_ of + FuzzTestPass data -> + ( expectation_ + , duration_ + , getAndClearDebugLogs False + ) + + FuzzTestFail data -> let - ( expectation_, duration_ ) = - runWithDuration (\() -> fuzzTest.thunk seed model.runInfo.fuzzRuns fuzzerInts) + newDebugLogs = + -- Unpause debug logs. + getAndClearDebugLogs False + |> (\_ -> + -- Collect debug logs from failing run. + data.rerunFailure () + |> (\() -> getAndClearDebugLogs False) + ) in - case expectation_ of - FuzzTestPass data -> - ( ( expectation_, duration_ ) - , getAndClearDebugLogs False - ) - - FuzzTestFail data -> - let - newDebugLogs = - getAndClearDebugLogs False - |> (\_ -> - data.rerunFailure () - |> (\() -> getAndClearDebugLogs False) - ) - in - ( ( expectation_, duration_ ) - , newDebugLogs - ) - ) - - hasDebugLogs = - not (isEmptyDebugLogs debugLogs) - - outcome = - case expectation of - FuzzTestPass { distributionReport } -> - Passed distributionReport - - FuzzTestFail { given, description, reason, distributionReport } -> - Failed - ( { given = given - , description = description - , reason = reason - } - , distributionReport - ) - - result : TestResult - result = - { labels = fuzzTest.labels - , outcome = outcome - , duration = duration - , hasDebugLogs = hasDebugLogs - } + ( expectation_ + , duration_ + , newDebugLogs + ) + ) + in + sendFuzzTestResult testId fuzzTest expectation duration debugLogs model.testReporter - report = - model.testReporter.reportComplete result - expectationElmCode = - if expectation == FuzzTestPass { distributionReport = NoDistribution } && not hasDebugLogs then - Nothing +sendFuzzTestResult : TestId -> FuzzTest -> FuzzTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg +sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = + let + jsDefinitionName = + fuzzTest.tag + + hasDebugLogs = + not (isEmptyDebugLogs debugLogs) + + outcome = + case expectation of + FuzzTestPass { distributionReport } -> + Passed distributionReport + + FuzzTestFail { given, description, reason, distributionReport } -> + Failed + ( { given = given + , description = description + , reason = reason + } + , distributionReport + ) + + result : TestResult + result = + { labels = fuzzTest.labels + , outcome = outcome + , duration = duration + , hasDebugLogs = hasDebugLogs + } - else - Debug.toString expectation - -- For `rerunFailure`: - |> String.replace "" "identity" - |> Just - in - ( model - , Ports.sendResult testId True jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report - ) + report = + testReporter.reportComplete result + + expectationElmCode = + if expectation == FuzzTestPass { distributionReport = NoDistribution } && not hasDebugLogs then + Nothing + + else + Debug.toString expectation + -- For `rerunFailure`: + |> String.replace "" "identity" + |> Just + in + Ports.sendResult testId True jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report update : Msg -> Model -> ( Model, Cmd Msg ) @@ -342,10 +315,10 @@ update msg ({ testReporter } as model) = Receive (Ok jsMessage) -> case jsMessage of RunUnitTest testId -> - dispatchUnitTest testId model + ( model, dispatchUnitTest testId model ) RunFuzzTest testId -> - dispatchFuzzTest testId model + ( model, dispatchFuzzTest testId model ) Summary duration failed todos -> let @@ -381,6 +354,142 @@ update msg ({ testReporter } as model) = Receive (Err err) -> ( model, Ports.sendError (Decode.errorToString err) ) + Trawl -> + case model.cacheTrawl of + NotTrawling -> + ( model, Cmd.none ) + + TrawlingUnitTests data -> + case Dict.get data.current model.unitTests of + Nothing -> + ( { model + | cacheTrawl = + TrawlingFuzzTests + { current = 0 + , unitTests = data.unitTests + , fuzzTests = [] + } + } + , trawlNext + ) + + Just unitTest -> + let + jsDefinitionName = + unitTest.tag + + hash = + getHash jsDefinitionName + + maybeCached = + Dict.get jsDefinitionName model.previousRun.cachedTests + |> Maybe.andThen + (\cachedTests -> + if hash == cachedTests.hash then + case Dict.get unitTest.labels cachedTests.unitTests of + -- As an optimization, passing unit tests without debug logs are not stored. + Nothing -> + Just ( UnitTestPass, noDebugLogs ) + + cached -> + cached + + else + Nothing + ) + in + case maybeCached of + Nothing -> + ( { model + | cacheTrawl = + TrawlingUnitTests + { current = data.current + 1 + , unitTests = data.current :: data.unitTests + } + } + , trawlNext + ) + + Just ( expectation, debugLogs ) -> + ( { model + | cacheTrawl = + TrawlingUnitTests + { current = data.current + 1 + , unitTests = data.unitTests + } + } + , Cmd.batch + [ trawlNext + , sendUnitTestResult data.current unitTest expectation 0 debugLogs model.testReporter + ] + ) + + TrawlingFuzzTests data -> + case Dict.get data.current model.fuzzTests of + Nothing -> + ( { model | cacheTrawl = NotTrawling } + , Ports.sendReady (List.reverse data.unitTests) (List.reverse data.fuzzTests) + ) + + Just fuzzTest -> + let + jsDefinitionName = + fuzzTest.tag + + hash = + getHash jsDefinitionName + + maybeCached = + Dict.get jsDefinitionName model.previousRun.cachedTests + |> Maybe.andThen + (\cachedTests -> + if + (hash == cachedTests.hash) + && (model.runInfo.initialSeed == model.previousRun.initialSeed) + -- If the fuzz tests specifies its own number of runs and the hash is the same, + -- then the number of runs must be unchanged. + && (fuzzTest.runs /= Nothing || model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) + then + case Dict.get fuzzTest.labels cachedTests.fuzzTests of + -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. + Nothing -> + Just ( FuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) + + cached -> + cached + + else + Nothing + ) + in + case maybeCached of + Nothing -> + ( { model + | cacheTrawl = + TrawlingFuzzTests + { current = data.current + 1 + , unitTests = data.unitTests + , fuzzTests = data.current :: data.fuzzTests + } + } + , trawlNext + ) + + Just ( expectation, debugLogs ) -> + ( { model + | cacheTrawl = + TrawlingFuzzTests + { current = data.current + 1 + , unitTests = data.unitTests + , fuzzTests = data.fuzzTests + } + } + , Cmd.batch + [ trawlNext + , sendFuzzTestResult data.current fuzzTest expectation 0 debugLogs model.testReporter + ] + ) + init : InitArgs -> Bool -> ( Model, Cmd Msg ) init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldSendBegin = @@ -419,20 +528,35 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldS , testReporter = testReporter , autoFail = autoFail , previousRun = previousRun + , cacheTrawl = + if shouldSendBegin then + TrawlingUnitTests + { current = 0 + , unitTests = [] + } + + else + NotTrawling } in ( model , if shouldSendBegin then - Ports.sendBegin - (Dict.size model.unitTests) - (Dict.size model.fuzzTests) - (model.testReporter.reportBegin model.runInfo) + Cmd.batch + [ Ports.sendBegin testCount + (model.testReporter.reportBegin model.runInfo) + , trawlNext + ] else Cmd.none ) +trawlNext : Cmd Msg +trawlNext = + Task.perform (\() -> Trawl) (Task.succeed ()) + + failInit : String -> Report -> Bool -> ( Model, Cmd Msg ) failInit message report _ = let @@ -454,6 +578,7 @@ failInit message report _ = , initialSeed = 0 , cachedTests = Dict.empty } + , cacheTrawl = NotTrawling } cmd = diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm index f927a005..f96bed6e 100644 --- a/elm/src/Test/Runner/Ports.elm +++ b/elm/src/Test/Runner/Ports.elm @@ -1,4 +1,4 @@ -port module Test.Runner.Ports exposing (JsMessage(..), receive, sendBegin, sendError, sendResult, sendSummary) +port module Test.Runner.Ports exposing (JsMessage(..), receive, sendBegin, sendError, sendReady, sendResult, sendSummary) import Json.Decode as Decode exposing (Decoder) import Json.Encode as Encode @@ -13,8 +13,8 @@ port elmTestPort__send : Decode.Value -> Cmd msg port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg -sendBegin : Int -> Int -> Maybe Decode.Value -> Cmd msg -sendBegin unitTests fuzzTests maybeReport = +sendBegin : Int -> Maybe Decode.Value -> Cmd msg +sendBegin testCount maybeReport = let extraFields = case maybeReport of @@ -28,13 +28,23 @@ sendBegin unitTests fuzzTests maybeReport = elmTestPort__send (Encode.object (( "type", Encode.string "BEGIN" ) - :: ( "unitTests", Encode.int unitTests ) - :: ( "fuzzTests", Encode.int fuzzTests ) + :: ( "testCount", Encode.int testCount ) :: extraFields ) ) +sendReady : List Int -> List Int -> Cmd msg +sendReady unitTests fuzzTests = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "READY" ) + , ( "unitTests", Encode.list Encode.int unitTests ) + , ( "fuzzTests", Encode.list Encode.int fuzzTests ) + ] + ) + + sendResult : Int -> Bool -> String -> List String -> Maybe String -> Decode.Value -> Decode.Value -> Cmd msg sendResult testId isFuzzTest jsDefinitionName labels expectationElmCode debugLogs report = elmTestPort__send diff --git a/lib/Supervisor.js b/lib/Supervisor.js index b72ebb02..7db27b0a 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -33,8 +33,10 @@ function run( watch ) { return new Promise(function (resolve) { - var unitTests = 0; - var fuzzTests = 0; + /** @type { Array | undefined } */ + var unitTests = undefined; + /** @type { Array | undefined } */ + var fuzzTests = undefined; var nextUnitTest = 0; var nextFuzzTest = 0; var finishedUnitTests = 0; @@ -184,6 +186,12 @@ function run( * @returns { void } */ function initWorker(socket) { + if (fuzzTests === undefined) { + throw new Error( + `fuzzTests is undefined, even though we have started workers for fuzz tests!` + ); + } + socket.setEncoding('utf8'); socket.setNoDelay(true); @@ -206,7 +214,7 @@ function run( send({ type: 'FUZZ', - testId: nextFuzzTest++, + testId: fuzzTests[nextFuzzTest++], }); } @@ -214,10 +222,14 @@ function run( * @typedef { | { type: 'BEGIN', - unitTests: number, - fuzzTests: number, + testCount: number, message?: any, } + | { + type: 'READY', + unitTests: Array, + fuzzTests: Array, + } | { type: 'RESULT', testId: number, @@ -296,9 +308,6 @@ function run( break; case 'BEGIN': - unitTests = response.unitTests; - fuzzTests = response.fuzzTests; - if (!Report.isMachineReadable(report)) { var headline = 'elm-test ' + elmTestVersion; var bar = '-'.repeat(headline.length); @@ -307,18 +316,41 @@ function run( } printResult(response.message); + break; - // If running multi-threaded, run fuzz tests on threads. - // Save one core for the main thread. - if (fuzzTests > 0 && processes > 1) { - startWorkers(Math.min(processes - 1, fuzzTests)); - } + case 'READY': + unitTests = response.unitTests; + fuzzTests = response.fuzzTests; - // Run unit tests in the main thread. - sendToMainProcess({ - type: 'UNIT', - testId: nextUnitTest++, - }); + if (unitTests.length === 0 && fuzzTests.length === 0) { + sendToMainProcess({ + type: 'SUMMARY', + duration: Date.now() - startingTime, + failures: failures, + todos: todos, + }); + } else { + // If running multi-threaded, run fuzz tests on threads. + // Save one core for the main thread. + if (fuzzTests.length > 0) { + if (processes > 1) { + startWorkers(Math.min(processes - 1, fuzzTests.length)); + } else if (unitTests.length === 0) { + sendToMainProcess({ + type: 'FUZZ', + testId: fuzzTests[nextFuzzTest++], + }); + } + } + + // Run unit tests in the main thread. + if (unitTests.length > 0) { + sendToMainProcess({ + type: 'UNIT', + testId: unitTests[nextUnitTest++], + }); + } + } break; case 'RESULT': { @@ -362,9 +394,14 @@ function run( break; } + if (unitTests === undefined || fuzzTests === undefined) { + // Not READY yet. + break; + } + if ( - finishedUnitTests >= unitTests && - finishedFuzzTests >= fuzzTests + finishedUnitTests >= unitTests.length && + finishedFuzzTests >= fuzzTests.length ) { sendToMainProcess({ type: 'SUMMARY', @@ -375,24 +412,24 @@ function run( } else { switch (response.testType) { case 'unit': - if (nextUnitTest < unitTests) { + if (nextUnitTest < unitTests.length) { send({ type: 'UNIT', - testId: nextUnitTest++, + testId: unitTests[nextUnitTest++], }); - } else if (processes === 1 && nextFuzzTest < fuzzTests) { + } else if (processes === 1 && nextFuzzTest < fuzzTests.length) { send({ type: 'FUZZ', - testId: nextFuzzTest++, + testId: fuzzTests[nextFuzzTest++], }); } break; case 'fuzz': - if (nextFuzzTest < fuzzTests) { + if (nextFuzzTest < fuzzTests.length) { send({ type: 'FUZZ', - testId: nextFuzzTest++, + testId: fuzzTests[nextFuzzTest++], }); } break; From 15b56640334bb5fd8097910d038d0827a96032db Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 11:38:58 +0200 Subject: [PATCH 58/81] Fix filtering of non-tests --- lib/Generate.js | 6 ++---- lib/Supervisor.js | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/Generate.js b/lib/Generate.js index 4c4fe6c0..cbd9771a 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -452,6 +452,7 @@ function ensurePreviousRunModule(previousRunModule) { * * @typedef { { hash: string, + isActuallyTest: boolean, unitTests: Array<{ labels: Array, expectation: string, debugLogs: Array }>, fuzzTests: Array<{ labels: Array, expectation: string, debugLogs: Array }>, } } CachedTests @@ -477,10 +478,7 @@ function generatePreviousRunModule(previousRunModule, previousRun) { const cachedTestsList = makeList( Object.entries(previousRun.cachedTests) - .filter( - ([, { unitTests, fuzzTests }]) => - unitTests.length > 0 || fuzzTests.length > 0 - ) + .filter(([, { isActuallyTest }]) => isActuallyTest) .map(([jsIdentifierName, { hash, unitTests, fuzzTests }]) => ` ( ${makeElmString(jsIdentifierName)} diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 7db27b0a..41a5999d 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -60,6 +60,8 @@ function run( for (var key in hashes) { toBePreviousRun.cachedTests[key] = { hash: hashes[key], + // We don’t know if exposed items are tests or not until runtime. + isActuallyTest: false, unitTests: [], fuzzTests: [], }; @@ -370,6 +372,7 @@ function run( const cachedTests = toBePreviousRun.cachedTests[response.jsDefinitionName]; + cachedTests.isActuallyTest = true; switch (response.testType) { case 'unit': finishedUnitTests++; From f0ee7c8f7cb8c4525000cd98751f2ab46aad09e2 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 11:49:59 +0200 Subject: [PATCH 59/81] elm-review --- elm/src/Test/Reporter/TestResults.elm | 2 -- elm/src/Test/Runner/Node.elm | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index 68047851..e08783f0 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -5,9 +5,7 @@ module Test.Reporter.TestResults exposing , TestResult ) -import Expect exposing (Expectation) import Test.Distribution exposing (DistributionReport) -import Test.Runner import Test.Runner.Failure exposing (Reason) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 5358ae1e..ac50f4e7 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -17,7 +17,6 @@ import Json.Decode as Decode import Json.Encode as Encode import Platform import Random -import Set exposing (Set) import Task import Test exposing (Test) import Test.Distribution exposing (DistributionReport(..)) @@ -203,9 +202,6 @@ dispatchFuzzTest testId model = jsDefinitionName = fuzzTest.tag - hash = - getHash jsDefinitionName - fuzzerInts = case Dict.get jsDefinitionName model.previousRun.cachedTests of Nothing -> @@ -236,7 +232,7 @@ dispatchFuzzTest testId model = runWithDuration (\() -> fuzzTest.thunk seed model.runInfo.fuzzRuns fuzzerInts) in case expectation_ of - FuzzTestPass data -> + FuzzTestPass _ -> ( expectation_ , duration_ , getAndClearDebugLogs False From 76ff7d9db41e9f3f986355e085e0010a46c4d230 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 11:53:49 +0200 Subject: [PATCH 60/81] Remove Debug.log --- example-application/tests/TestsPassing.elm | 1 - 1 file changed, 1 deletion(-) diff --git a/example-application/tests/TestsPassing.elm b/example-application/tests/TestsPassing.elm index f72205c6..9500d5fd 100644 --- a/example-application/tests/TestsPassing.elm +++ b/example-application/tests/TestsPassing.elm @@ -10,7 +10,6 @@ testEqual = test "Expect.equal works" <| \() -> Something.ultimateAnswer - |> Debug.log "HÄR" |> Expect.equal 42 From 4f9dc4a80c68cd5301aa4f7ca1b6dfb0dd0c6b9e Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 11:56:48 +0200 Subject: [PATCH 61/81] Resolve TODO comment --- elm/src/Test/Runner/Node.elm | 1 - 1 file changed, 1 deletion(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index ac50f4e7..7d04f8fb 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -578,7 +578,6 @@ failInit message report _ = } cmd = - -- TODO: This isn't using the reporter? How does that work? Ports.sendSummary 1 (Encode.string message) in ( model, cmd ) From 060ff9715b6594863a07efbfc371cb924ebcb537 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 12:14:41 +0200 Subject: [PATCH 62/81] Fix crash when fuzz tests are too fast --- lib/Supervisor.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 41a5999d..a4493fda 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -194,6 +194,11 @@ function run( ); } + // Other workers might have exhausted all fuzz tests before this one even got a chance to start. + if (nextFuzzTest >= fuzzTests.length) { + return; + } + socket.setEncoding('utf8'); socket.setNoDelay(true); From 00fd2488140e6f211ad2ba42457903b67a3dd553 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 13:00:07 +0200 Subject: [PATCH 63/81] Resolve TODO --- lib/Generate.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/Generate.js b/lib/Generate.js index cbd9771a..e360d238 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -128,7 +128,6 @@ function patch(hashes, content) { ` var _Debug_logs = []; var _Debug_logPaused = false; -// TODO: Debug.todo does not work like I thought it would. Not a good tip as is. var _Debug_logPausedMessage = 'For passing fuzz tests, Debug.log is not shown, since showing logs from lots of runs is pretty confusing. Tip: Use Debug.todo to fail a test from anywhere.'; var _Debug_log = F2(function(tag, value) { From 6bae8f560b9a2b3ddb550126b367706f4ac57f8f Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 13:09:14 +0200 Subject: [PATCH 64/81] Also expose durationFloat --- elm/src/Test/Reporter/Json.elm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elm/src/Test/Reporter/Json.elm b/elm/src/Test/Reporter/Json.elm index 165f2cd5..2b200ea2 100644 --- a/elm/src/Test/Reporter/Json.elm +++ b/elm/src/Test/Reporter/Json.elm @@ -29,8 +29,10 @@ reportComplete { duration, labels, outcome } = , ( "failures", Encode.list identity (encodeFailures outcome) ) , ( "distributionReports", Encode.list identity (encodeDistributionReports outcome) ) - -- We have a more exact float these days, but to avoid a breaking change we round it to an int. + -- Keep the "duration" field Int for backwards compatibility, + -- and also expose the new Float field for more precision. , ( "duration", Encode.string <| String.fromInt (round duration) ) + , ( "durationFloat", Encode.string <| String.fromFloat duration ) ] From 476565c2ef76595cdfcbce30e37856b630aa0ea9 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 13:51:16 +0200 Subject: [PATCH 65/81] Add --unbuffered-logs --- README.md | 16 ++++++++++++++++ lib/Generate.js | 32 ++++++++++++++++++++++++++------ lib/Hash.js | 6 +++++- lib/RunTests.js | 7 +++++-- lib/Supervisor.js | 10 +++++++--- lib/elm-test.js | 5 +++++ 6 files changed, 64 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ec510fcb..0563fab3 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,22 @@ Start the runner in watch mode. Your tests will automatically rerun whenever you elm-test --watch +### --no-clear-console + +By default, the console is cleared before each run in watch mode, so you only see the latest information. If you don’t like this, turn it off with `--no-clear-console`. + + elm-test --watch --no-clear-console + +### --unbuffered-logs + +elm-test collects all `Debug.log` output while executing a test, and displays it all once the test in question is finished. This way elm-test can print _which_ test the logs came from. + +If the function your are testing gets into an infinite loop, it means that your debug logs will never show up. Then it can be useful to have the logs print _immediately_ instead (at the loss of no longer being able to label which tests the logs came from). To avoid confusion, use [Test.only](https://package.elm-lang.org/packages/elm-explorations/test/latest/Test#only) to isolate your test, or pass `--workers 1` to run in single-threaded mode to avoid oddly mixed output: + + elm-test --unbuffered-logs --workers 1 + +For _failing_ fuzz tests, elm-test only prints `Debug.log` output from the run of the fuzz test that produced the failure, which is usually what you want to debug. (Earlier, passing runs of the function with different input is just noise). For _passing_ fuzz tests, elm-test _ignores_ your `Debug.log` calls (and instead displays a note about this). Let’s imagine you are debugging a failing fuzz test. After a while it finally passes. There is no longer a failing run, so which one should we pick logs from? All of them? But would you really like to see the screen fill with 100+ repetitions of your logs at that point? Probably not. But if you actually _do_ want to show logs from all runs, you can use `--unbuffered-logs` for this use case, too. Also remember that you can make the test fail from anywhere using `Debug.todo` – that’s also a way to make logs appear! + ### --seed Run with a specific fuzzer seed, rather than a randomly generated seed. This allows reproducing a failing fuzz-test. The command needed to reproduce (including the `--seed` flag) is printed after each test run. Copy, paste and run it! diff --git a/lib/Generate.js b/lib/Generate.js index e360d238..c548e011 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -22,9 +22,15 @@ const after = fs.readFileSync( }> } testModules * @param { string } pipeFilename * @param { string } dest + * @param { boolean } unbufferedLogs * @returns { Record } */ -function prepareCompiledJsFile(testModules, pipeFilename, dest) { +function prepareCompiledJsFile( + testModules, + pipeFilename, + dest, + unbufferedLogs +) { const content = fs.readFileSync(dest, 'utf8'); const names = testModules.flatMap((mod) => @@ -33,12 +39,12 @@ function prepareCompiledJsFile(testModules, pipeFilename, dest) { ) ); - const hashes = Hash.calculateHashes(names, content); + const hashes = Hash.calculateHashes(unbufferedLogs, names, content); const finalContent = ` ${before} var Elm = (function() { -${patch(hashes, content)} +${patch(hashes, unbufferedLogs, content)} return this.Elm; }).call({}); var pipeFilename = ${JSON.stringify(pipeFilename)}; @@ -109,10 +115,11 @@ function toCompiledJavaScriptName(moduleName, valueName) { * and the first usage of `console.warn`. * * @param { Record } hashes + * @param { boolean } unbufferedLogs * @param { string } content * @returns { string } */ -function patch(hashes, content) { +function patch(hashes, unbufferedLogs, content) { return ( 'var __elmTestSymbol = Symbol("elmTestSymbol");\n' + content @@ -125,7 +132,20 @@ function patch(hashes, content) { .replace('console.warn', '') .replace( debugLogDefinition, - ` + unbufferedLogs + ? ` +var _Debug_logs = []; +var _Debug_logPaused = false; +var _Debug_log = F2(function(tag, value) +{ + if (_Debug_logs.length === 0) { + _Debug_logs.push(''); + } + console.error(tag + ': ' + _Debug_toString(value)); + return value; +}); + `.trim() + : ` var _Debug_logs = []; var _Debug_logPaused = false; var _Debug_logPausedMessage = 'For passing fuzz tests, Debug.log is not shown, since showing logs from lots of runs is pretty confusing. Tip: Use Debug.todo to fail a test from anywhere.'; @@ -140,7 +160,7 @@ var _Debug_log = F2(function(tag, value) } return value; }); - `.trim() + `.trim() ) .replace( getAndClearDebugLogsDefinition, diff --git a/lib/Hash.js b/lib/Hash.js index 63d1aa24..6857368d 100644 --- a/lib/Hash.js +++ b/lib/Hash.js @@ -12,12 +12,16 @@ const Tarjan = require('./Tarjan'); * This way we can tell if the code that will be running via an exposed `Test` * value has changed or not, and thus if we need to re-run it or not. * + * @param { boolean } unbufferedLogs * @param { Array } names * @param { string } code * @returns { Record } */ -function calculateHashes(names, code) { +function calculateHashes(unbufferedLogs, names, code) { const chunks = parseStep(code); + if (unbufferedLogs) { + chunks['_Debug_log'] += '/* unbuffered */'; + } const graph = graphStep(chunks); makeAcyclicStep(graph); return hashStep(names, chunks, graph); diff --git a/lib/RunTests.js b/lib/RunTests.js index 2a741b17..c389d401 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -141,6 +141,7 @@ function watcherEventMessage(queue) { * @typedef { { watch: boolean, clearConsole: boolean, + unbufferedLogs: boolean, report: import('./Report').Report, seed: number, fuzz: number, @@ -160,7 +161,7 @@ function runTests( pathToElmBinary, testFileGlobs, processes, - { watch, clearConsole, report, seed, fuzz } + { watch, clearConsole, unbufferedLogs, report, seed, fuzz } ) { /** @type { import('chokidar').FSWatcher | undefined } */ let watcher = undefined; @@ -275,7 +276,8 @@ function runTests( const hashes = Generate.prepareCompiledJsFile( testModules, pipeFilename, - dest + dest, + unbufferedLogs ); progressLogger.log('Starting tests'); @@ -290,6 +292,7 @@ function runTests( seed, report, processes, + unbufferedLogs, dest, watch ); diff --git a/lib/Supervisor.js b/lib/Supervisor.js index a4493fda..d473ee48 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -16,6 +16,7 @@ const XMLBuilder = require('./XMLBuilder'); * @param { number } seed * @param { import('./Report').Report } report * @param { number } processes + * @param { boolean } unbufferedLogs * @param { string } dest * @param { boolean } watch * @returns { Promise } @@ -29,6 +30,7 @@ function run( seed, report, processes, + unbufferedLogs, dest, watch ) { @@ -363,7 +365,7 @@ function run( case 'RESULT': { handleResult(response.testId, response.message); printResult(response.message); - if (response.debugLogs.length > 0) { + if (response.debugLogs.length > 0 && !unbufferedLogs) { if (report !== 'console') { console.error(response.labels.slice().reverse().join(' > ')); } @@ -378,10 +380,12 @@ function run( const cachedTests = toBePreviousRun.cachedTests[response.jsDefinitionName]; cachedTests.isActuallyTest = true; + const usedUnbufferedLogs = + unbufferedLogs && response.debugLogs.length > 0; switch (response.testType) { case 'unit': finishedUnitTests++; - if (response.expectationElmCode !== null) { + if (response.expectationElmCode !== null && !usedUnbufferedLogs) { cachedTests.unitTests.push({ labels: response.labels, expectation: response.expectationElmCode, @@ -392,7 +396,7 @@ function run( case 'fuzz': finishedFuzzTests++; - if (response.expectationElmCode !== null) { + if (response.expectationElmCode !== null && !usedUnbufferedLogs) { cachedTests.fuzzTests.push({ labels: response.labels, expectation: response.expectationElmCode, diff --git a/lib/elm-test.js b/lib/elm-test.js index 9a879602..04837fb3 100644 --- a/lib/elm-test.js +++ b/lib/elm-test.js @@ -162,6 +162,11 @@ function main() { '--no-clear-console', "Don't clear the console when running with --watch" ) + .option( + '--unbuffered-logs', + 'Print debug logs immediately instead of at the end of each test', + false + ) // For example `--seed` and `--fuzz` only make sense for the “tests” command // and could be specified for that command only, but then they won’t show up // in `--help`. From 40b9fdf955bf7e97493c9b799e1aeaf9f32893e4 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 14:11:54 +0200 Subject: [PATCH 66/81] Update --workers docs --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0563fab3..633254cb 100644 --- a/README.md +++ b/README.md @@ -189,16 +189,18 @@ Define how many times each fuzz-test should run. Defaults to `100`. ### --workers -Choose how many workers elm-test should use to run tests in parallel. Defaults to the number of “logical CPU cores” of the machine you run the tests on. +Choose how many workers elm-test should use to run fuzz tests in parallel. Defaults to the number of “logical CPU cores” of the machine you run the tests on. elm-test --workers 4 -Your computer might say that it has 12 logical CPU cores. Then dividing up the tests between 12 parallel workers is the theoretical optimum for running the tests as quickly as possible. But in practice your tests might run faster with just 4 workers in parallel due to overhead. Play around with it and see what is the fastest for your test suite on your computer! +Your computer might say that it has 12 logical CPU cores. Then dividing up the fuzz tests between 12 parallel workers is the theoretical optimum for running the tests as quickly as possible. But in practice your tests might run faster with just 4 workers in parallel due to overhead. Play around with it and see what is the fastest for your test suite on your computer! To see the number of logical CPU cores on your machine, run `node -p "os.cpus().length"` (it’s also shown in `elm-test --help`). If you pass `--workers 1`, elm-test won’t even start a new thread for running the tests in – it’ll do everything in the main thread (single-threaded mode). +Currently, elm-test always executes unit tests on the main thread, and only uses separate threads for fuzz tests. Unit tests tend to execute so fast that the overhead of threads isn’t worth it. But fuzz tests often run long enough to benefit from parallelization. + ### --report Specify which format to use for reporting test results. Valid options are: From 939026fe7a91badd39cde0867ebc2af3634075bd Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 18:27:43 +0200 Subject: [PATCH 67/81] Better default random seed --- lib/elm-test.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/elm-test.js b/lib/elm-test.js index 04837fb3..e65943a8 100644 --- a/lib/elm-test.js +++ b/lib/elm-test.js @@ -172,7 +172,15 @@ function main() { // in `--help`. .addOption( new Option('--seed ', 'Run with a specific fuzzer seed') - .default(Math.floor(Math.random() * 407199254740991) + 1000, 'random') + // This will be passed to `Random.initialSeed`, which calls: + // `Bitwise.shiftRightZfBy 0 (incr + seed)` where `seed` is + // our number and `incr` is a constant. `Bitwise.shiftRightZfBy 0` + // is basically the same as as `modBy 0x100000000`. `incr` just shifts + // the numbers, it doesn’t affect how many different seeds there can be. + // In other words, there is no reason to pass a number higher than + // or equal to 0x100000000: After that we are just repeating seeds + // and have to be careful to not make some seeds more likely than others. + .default(Math.floor(Math.random() * 0x100000000), 'random') .argParser(parsePositiveInteger(0)) ) .option( From a7cdf5dc7e772a7ba8490d0dd87a07f09f24e04b Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 18:43:41 +0200 Subject: [PATCH 68/81] Buffer debug logs before tests start --- elm/src/Test/Runner/Node.elm | 10 +++++++++- elm/src/Test/Runner/Ports.elm | 5 +++-- lib/Supervisor.js | 9 +++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 7d04f8fb..058c6a70 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -534,11 +534,19 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldS else NotTrawling } + + -- In the main thread, we log these. + -- In workers, we just clear them and ignore them – + -- they are identical to the main thread. + debugLogs = + getAndClearDebugLogs False in ( model , if shouldSendBegin then Cmd.batch - [ Ports.sendBegin testCount + [ Ports.sendBegin + testCount + debugLogs (model.testReporter.reportBegin model.runInfo) , trawlNext ] diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm index f96bed6e..8460956a 100644 --- a/elm/src/Test/Runner/Ports.elm +++ b/elm/src/Test/Runner/Ports.elm @@ -13,8 +13,8 @@ port elmTestPort__send : Decode.Value -> Cmd msg port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg -sendBegin : Int -> Maybe Decode.Value -> Cmd msg -sendBegin testCount maybeReport = +sendBegin : Int -> Decode.Value -> Maybe Decode.Value -> Cmd msg +sendBegin testCount debugLogs maybeReport = let extraFields = case maybeReport of @@ -29,6 +29,7 @@ sendBegin testCount maybeReport = (Encode.object (( "type", Encode.string "BEGIN" ) :: ( "testCount", Encode.int testCount ) + :: ( "debugLogs", debugLogs ) :: extraFields ) ) diff --git a/lib/Supervisor.js b/lib/Supervisor.js index d473ee48..36aa0084 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -232,6 +232,7 @@ function run( | { type: 'BEGIN', testCount: number, + debugLogs: Array, message?: any, } | { @@ -325,6 +326,14 @@ function run( } printResult(response.message); + if (response.debugLogs.length > 0) { + for (const debugLog of response.debugLogs) { + console.error(debugLog); + } + if (report === 'console') { + console.error('\n'); + } + } break; case 'READY': From 7d60960eecc9dd4926055d0acc5bdb468961f3cf Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 2 Aug 2026 21:08:04 +0200 Subject: [PATCH 69/81] Use same seed as previous time if there was a fuzz failure --- README.md | 2 + elm/src/Test/Runner/Node.elm | 132 ++++++++++++++++++++++------------ elm/src/Test/Runner/Ports.elm | 4 +- lib/Generate.js | 26 ++++++- lib/RunTests.js | 3 +- lib/Supervisor.js | 12 ++-- lib/elm-test.js | 10 +-- 7 files changed, 122 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 633254cb..de10f1fe 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,8 @@ Run with a specific fuzzer seed, rather than a randomly generated seed. This all elm-test --seed 336948560956134 +On top of that, if you run elm-test without the `--seed` flag, elm-test will automatically use the same seed as the last run if there was a fuzz test failure, letting you reproduce errors without doing anything. It even tries to fast-forward you through the fuzzing. So if it took some time for the fuzzer to find the problem the first time, the next run should be instant. + ### --fuzz Define how many times each fuzz-test should run. Defaults to `100`. diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 058c6a70..5eb3bafe 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,4 @@ -module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun) +module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun, SeedChoice(..)) {-| @@ -8,7 +8,7 @@ module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun) Runs a test and outputs its results to the console. Exit code is 0 if tests passed and 2 if any failed. Returns 1 if something went wrong. -@docs checkTagged, run, TestProgram, PreviousRun +@docs checkTagged, run, TestProgram, PreviousRun, SeedChoice -} @@ -43,19 +43,8 @@ type alias DebugLogs = Decode.Value -type alias InitArgs = - { initialSeed : Int - , globs : List String - , paths : List String - , fuzzRuns : Int - , tests : Tests - , report : Report - , previousRun : PreviousRun - } - - type alias RunnerOptions = - { seed : Int + { seed : SeedChoice , runs : Int , report : Report , globs : List String @@ -64,6 +53,14 @@ type alias RunnerOptions = } +type SeedChoice + = UserSuppliedSeed Int + -- Note: We might not use the random seed here; + -- we might end up using the same seed as the previous run instead + -- if that reproduces a fuzz test failure. + | RandomSeed Int + + type alias Model = { unitTests : Dict TestId UnitTest , fuzzTests : Dict TestId FuzzTest @@ -203,22 +200,35 @@ dispatchFuzzTest testId model = fuzzTest.tag fuzzerInts = - case Dict.get jsDefinitionName model.previousRun.cachedTests of - Nothing -> - [] - - Just cachedTests -> - case Dict.get fuzzTest.labels cachedTests.fuzzTests of - Nothing -> - [] + if + -- In `init` we use the same seed as the previous run if there was a failing fuzz test. + -- If the user has explicitly passed a different seed, don’t try to reproduce the previous + -- failure. They are clearly trying to run something else. + (model.runInfo.initialSeed == model.previousRun.initialSeed) + -- The number of fuzz runs must be the same (or more) as the previous run – otherwise + -- the user has explicitly passed fewer, and we can’t know if the previous failure + -- would hit or not. Since the seed is still the same, there’s still a chance it will. + && (model.runInfo.fuzzRuns >= model.previousRun.fuzzRuns) + then + case Dict.get jsDefinitionName model.previousRun.cachedTests of + Nothing -> + [] + + Just cachedTests -> + case Dict.get fuzzTest.labels cachedTests.fuzzTests of + Nothing -> + [] + + Just ( expectation_, _ ) -> + case expectation_ of + FuzzTestPass _ -> + [] + + FuzzTestFail data -> + data.fuzzerInts - Just ( expectation_, _ ) -> - case expectation_ of - FuzzTestPass _ -> - [] - - FuzzTestFail data -> - data.fuzzerInts + else + [] seed = Random.initialSeed model.runInfo.initialSeed @@ -487,8 +497,8 @@ update msg ({ testReporter } as model) = ) -init : InitArgs -> Bool -> ( Model, Cmd Msg ) -init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldSendBegin = +init : RunnerOptions -> Tests -> Bool -> ( Model, Cmd Msg ) +init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = let autoFail = case ( tests.seenOnly, tests.seenSkip ) of @@ -510,6 +520,18 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldS testReporter = createReporter report + initialSeed = + case seed of + UserSuppliedSeed seed_ -> + seed_ + + RandomSeed seed_ -> + if previousRunHasFailingFuzzTest previousRun tests.fuzzTests then + previousRun.initialSeed + + else + seed_ + model : Model model = { unitTests = toIndexedDict tests.unitTests @@ -518,7 +540,7 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldS { testCount = testCount , globs = globs , paths = paths - , fuzzRuns = fuzzRuns + , fuzzRuns = runs , initialSeed = initialSeed } , testReporter = testReporter @@ -545,7 +567,7 @@ init { globs, paths, fuzzRuns, initialSeed, report, tests, previousRun } shouldS , if shouldSendBegin then Cmd.batch [ Ports.sendBegin - testCount + initialSeed debugLogs (model.testReporter.reportBegin model.runInfo) , trawlNext @@ -591,6 +613,33 @@ failInit message report _ = ( model, cmd ) +previousRunHasFailingFuzzTest : PreviousRun -> List FuzzTest -> Bool +previousRunHasFailingFuzzTest previousRun = + List.any + (\fuzzTest -> + let + jsDefinitionName = + fuzzTest.tag + in + case Dict.get jsDefinitionName previousRun.cachedTests of + Nothing -> + False + + Just cachedTests -> + case Dict.get fuzzTest.labels cachedTests.fuzzTests of + Nothing -> + False + + Just ( expectation_, _ ) -> + case expectation_ of + FuzzTestPass _ -> + False + + FuzzTestFail _ -> + True + ) + + toIndexedDict : List a -> Dict Int a toIndexedDict list = list @@ -651,7 +700,7 @@ placeholderReplaceMe___ name = {-| Run the tests. -} run : RunnerOptions -> List ( String, List (Maybe Test) ) -> TestProgram -run { runs, seed, report, globs, paths, previousRun } possiblyTests = +run options possiblyTests = let testsList = possiblyTests @@ -670,7 +719,7 @@ run { runs, seed, report, globs, paths, previousRun } possiblyTests = in if List.isEmpty testsList then Platform.worker - { init = failInit (noTestsFoundError globs) report + { init = failInit (noTestsFoundError options.globs) options.report , update = \_ model -> ( model, Cmd.none ) , subscriptions = \_ -> Sub.none } @@ -679,20 +728,9 @@ run { runs, seed, report, globs, paths, previousRun } possiblyTests = let tests = Test.Runner.toTests (Test.concat testsList) - - wrappedInit = - init - { initialSeed = seed - , globs = globs - , paths = paths - , fuzzRuns = runs - , tests = tests - , report = report - , previousRun = previousRun - } in Platform.worker - { init = wrappedInit + { init = init options tests , update = update , subscriptions = \_ -> Ports.receive Receive } diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm index 8460956a..6aa3c029 100644 --- a/elm/src/Test/Runner/Ports.elm +++ b/elm/src/Test/Runner/Ports.elm @@ -14,7 +14,7 @@ port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg sendBegin : Int -> Decode.Value -> Maybe Decode.Value -> Cmd msg -sendBegin testCount debugLogs maybeReport = +sendBegin initialSeed debugLogs maybeReport = let extraFields = case maybeReport of @@ -28,7 +28,7 @@ sendBegin testCount debugLogs maybeReport = elmTestPort__send (Encode.object (( "type", Encode.string "BEGIN" ) - :: ( "testCount", Encode.int testCount ) + :: ( "initialSeed", Encode.int initialSeed ) :: ( "debugLogs", debugLogs ) :: extraFields ) diff --git a/lib/Generate.js b/lib/Generate.js index c548e011..2ae82b10 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -280,7 +280,7 @@ function getModule(generatedCodeDir, moduleName) { /** * @param { number } fuzz - * @param { number } seed + * @param { number | null } seed * @param { import('./Report').Report } report * @param { Array } testFileGlobs * @param { Array } testFilePaths @@ -397,7 +397,7 @@ function indentAllButFirstLine(indent, string) { /** * @param { number } fuzz - * @param { number } seed + * @param { number | null } seed * @param { import('./Report').Report } report * @param { Array } testFileGlobs * @param { Array } testFilePaths @@ -407,7 +407,11 @@ function makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) { return ` { runs = ${fuzz} , report = ${generateElmReportVariant(report)} -, seed = ${seed} +, seed = ${ + seed === null + ? `Test.Runner.Node.RandomSeed ${makeRandomSeed()}` + : `Test.Runner.Node.UserSuppliedSeed ${seed}` + } , previousRun = ${previousRunModuleName.join('.')}.previousRun , globs = ${indentAllButFirstLine(' ', makeList(testFileGlobs.map(makeElmString)))} @@ -417,6 +421,22 @@ function makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) { `.trim(); } +/** + * This will be passed to `Random.initialSeed`, which calls: + * `Bitwise.shiftRightZfBy 0 (incr + seed)` where `seed` is + * our number and `incr` is a constant. `Bitwise.shiftRightZfBy 0` + * is basically the same as as `modBy 0x100000000`. `incr` just shifts + * the numbers, it doesn’t affect how many different seeds there can be. + * In other words, there is no reason to pass a number higher than + * or equal to 0x100000000: After that we are just repeating seeds + * and have to be careful to not make some seeds more likely than others. + * + * @returns { number } + */ +function makeRandomSeed() { + return Math.floor(Math.random() * 0x100000000); +} + /** * @param { import('./Report').Report } report * @returns { string } diff --git a/lib/RunTests.js b/lib/RunTests.js index c389d401..ba3646be 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -143,7 +143,7 @@ function watcherEventMessage(queue) { clearConsole: boolean, unbufferedLogs: boolean, report: import('./Report').Report, - seed: number, + seed: number | null, fuzz: number, } } Options @@ -289,7 +289,6 @@ function runTests( previousRunModule, pipeFilename, fuzz, - seed, report, processes, unbufferedLogs, diff --git a/lib/Supervisor.js b/lib/Supervisor.js index 36aa0084..82f2c3ee 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -13,7 +13,6 @@ const XMLBuilder = require('./XMLBuilder'); * @param { import('./Generate').Module } previousRunModule * @param { string } pipeFilename * @param { number } fuzz - * @param { number } seed * @param { import('./Report').Report } report * @param { number } processes * @param { boolean } unbufferedLogs @@ -27,7 +26,6 @@ function run( previousRunModule, pipeFilename, fuzz, - seed, report, processes, unbufferedLogs, @@ -56,7 +54,10 @@ function run( /** @type { import('./Generate').PreviousRun } */ var toBePreviousRun = { fuzzRuns: fuzz, - initialSeed: seed, + // When running with a random seed, Node.elm might decide to use + // the same seed as the last run to reproduce a failure. + // This is replaced with the real value at BEGIN. + initialSeed: -1, cachedTests: {}, }; for (var key in hashes) { @@ -231,7 +232,7 @@ function run( * @typedef { | { type: 'BEGIN', - testCount: number, + initialSeed: number, debugLogs: Array, message?: any, } @@ -318,6 +319,9 @@ function run( break; case 'BEGIN': + // Store the seed actually chosen to be used in the end. + toBePreviousRun.initialSeed = response.initialSeed; + if (!Report.isMachineReadable(report)) { var headline = 'elm-test ' + elmTestVersion; var bar = '-'.repeat(headline.length); diff --git a/lib/elm-test.js b/lib/elm-test.js index e65943a8..3ca9d11a 100644 --- a/lib/elm-test.js +++ b/lib/elm-test.js @@ -172,15 +172,7 @@ function main() { // in `--help`. .addOption( new Option('--seed ', 'Run with a specific fuzzer seed') - // This will be passed to `Random.initialSeed`, which calls: - // `Bitwise.shiftRightZfBy 0 (incr + seed)` where `seed` is - // our number and `incr` is a constant. `Bitwise.shiftRightZfBy 0` - // is basically the same as as `modBy 0x100000000`. `incr` just shifts - // the numbers, it doesn’t affect how many different seeds there can be. - // In other words, there is no reason to pass a number higher than - // or equal to 0x100000000: After that we are just repeating seeds - // and have to be careful to not make some seeds more likely than others. - .default(Math.floor(Math.random() * 0x100000000), 'random') + .default(null, 'random') .argParser(parsePositiveInteger(0)) ) .option( From c4fd5a13670cd4b76426f1e12673476e46c8ccec Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Wed, 5 Aug 2026 18:39:01 +0200 Subject: [PATCH 70/81] Use Array instead of Dict for tests --- elm/src/Test/Runner/Node.elm | 75 ++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 5eb3bafe..37cf7198 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -12,6 +12,7 @@ passed and 2 if any failed. Returns 1 if something went wrong. -} +import Array exposing (Array) import Dict exposing (Dict) import Json.Decode as Decode import Json.Encode as Encode @@ -62,8 +63,8 @@ type SeedChoice type alias Model = - { unitTests : Dict TestId UnitTest - , fuzzTests : Dict TestId FuzzTest + { unitTests : Array UnitTest + , fuzzTests : Array FuzzTest , runInfo : RunInfo , testReporter : TestReporter , autoFail : Maybe String @@ -126,7 +127,7 @@ isEmptyDebugLogs debugLogs = dispatchUnitTest : TestId -> Model -> Cmd Msg dispatchUnitTest testId model = - case Dict.get testId model.unitTests of + case Array.get testId model.unitTests of Nothing -> Ports.sendError ("Unit test not found: " ++ String.fromInt testId) @@ -190,7 +191,7 @@ sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = dispatchFuzzTest : TestId -> Model -> Cmd Msg dispatchFuzzTest testId model = - case Dict.get testId model.fuzzTests of + case Array.get testId model.fuzzTests of Nothing -> Ports.sendError ("Fuzz test not found: " ++ String.fromInt testId) @@ -366,7 +367,7 @@ update msg ({ testReporter } as model) = ( model, Cmd.none ) TrawlingUnitTests data -> - case Dict.get data.current model.unitTests of + case Array.get data.current model.unitTests of Nothing -> ( { model | cacheTrawl = @@ -431,7 +432,7 @@ update msg ({ testReporter } as model) = ) TrawlingFuzzTests data -> - case Dict.get data.current model.fuzzTests of + case Array.get data.current model.fuzzTests of Nothing -> ( { model | cacheTrawl = NotTrawling } , Ports.sendReady (List.reverse data.unitTests) (List.reverse data.fuzzTests) @@ -515,7 +516,7 @@ init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = Just "Test.only and Test.skip were used" testCount = - List.length tests.unitTests + List.length tests.fuzzTests + Array.length tests.unitTests + Array.length tests.fuzzTests testReporter = createReporter report @@ -534,8 +535,8 @@ init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = model : Model model = - { unitTests = toIndexedDict tests.unitTests - , fuzzTests = toIndexedDict tests.fuzzTests + { unitTests = tests.unitTests + , fuzzTests = tests.fuzzTests , runInfo = { testCount = testCount , globs = globs @@ -588,8 +589,8 @@ failInit message report _ = let model : Model model = - { unitTests = Dict.empty - , fuzzTests = Dict.empty + { unitTests = Array.empty + , fuzzTests = Array.empty , runInfo = { testCount = 0 , globs = [] @@ -613,38 +614,36 @@ failInit message report _ = ( model, cmd ) -previousRunHasFailingFuzzTest : PreviousRun -> List FuzzTest -> Bool +previousRunHasFailingFuzzTest : PreviousRun -> Array FuzzTest -> Bool previousRunHasFailingFuzzTest previousRun = - List.any - (\fuzzTest -> - let - jsDefinitionName = - fuzzTest.tag - in - case Dict.get jsDefinitionName previousRun.cachedTests of - Nothing -> - False + Array.foldl + (\fuzzTest hasFailingFuzzTest -> + if hasFailingFuzzTest then + hasFailingFuzzTest - Just cachedTests -> - case Dict.get fuzzTest.labels cachedTests.fuzzTests of - Nothing -> - False + else + let + jsDefinitionName = + fuzzTest.tag + in + case Dict.get jsDefinitionName previousRun.cachedTests of + Nothing -> + False + + Just cachedTests -> + case Dict.get fuzzTest.labels cachedTests.fuzzTests of + Nothing -> + False - Just ( expectation_, _ ) -> - case expectation_ of - FuzzTestPass _ -> - False + Just ( expectation_, _ ) -> + case expectation_ of + FuzzTestPass _ -> + False - FuzzTestFail _ -> - True + FuzzTestFail _ -> + True ) - - -toIndexedDict : List a -> Dict Int a -toIndexedDict list = - list - |> List.indexedMap Tuple.pair - |> Dict.fromList + False checkTagged : a -> JsDefinitionName -> Maybe Test From 46944a172af26ba9fe93727415dd75a81cf875ba Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Wed, 5 Aug 2026 18:43:20 +0200 Subject: [PATCH 71/81] Some comments --- elm/src/Test/Runner/Node.elm | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 37cf7198..82eaa21b 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -32,14 +32,24 @@ import Test.Runner.Ports as Ports exposing (JsMessage(..)) -- TYPES +{-| A `TestId` is just an index into an `Array` of tests. +-} type alias TestId = Int +{-| The compiled JavaScript name of an exposed value, +such as `$user$project$Tests$suite`. +-} type alias JsDefinitionName = String +{-| Collected `Debug.log`s during a test (or during initialization before running any tests). +There are stored as `Decode.Value` instead of `List String` as an optimization. They are +collected in JavaScript code, given to Elm for a short while, and then sent through a port. +So going to from a JS array, to an Elm list, back to a JS array is pretty wasteful. +-} type alias DebugLogs = Decode.Value From c0d9c68fc8a2b08c2bcac9c1a816f59a6199d75d Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 16 Aug 2026 01:06:25 +0200 Subject: [PATCH 72/81] Fix hack for rerunFailure --- elm/src/Test/Runner/Node.elm | 55 ++++++++++++++++++++++++------------ lib/Generate.js | 4 +-- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 82eaa21b..ac3761d2 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,7 @@ -module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun, SeedChoice(..)) +module Test.Runner.Node exposing + ( checkTagged, run, TestProgram, PreviousRun, SeedChoice(..) + , CachedFuzzTestExpectation(..) + ) {-| @@ -97,10 +100,23 @@ type alias CachedTests = , unitTests : Dict (List String) ( UnitTestExpectation, DebugLogs ) -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. - , fuzzTests : Dict (List String) ( FuzzTestExpectation, DebugLogs ) + , fuzzTests : Dict (List String) ( CachedFuzzTestExpectation, DebugLogs ) } +{-| Same as `FuzzTestExpectation`, but without `rerunFailure`. +-} +type CachedFuzzTestExpectation + = CachedFuzzTestPass { distributionReport : DistributionReport } + | CachedFuzzTestFail + { given : Maybe String + , fuzzerInts : List Int + , description : String + , reason : Reason + , distributionReport : DistributionReport + } + + type CacheTrawl = NotTrawling | TrawlingUnitTests @@ -232,10 +248,10 @@ dispatchFuzzTest testId model = Just ( expectation_, _ ) -> case expectation_ of - FuzzTestPass _ -> + CachedFuzzTestPass _ -> [] - FuzzTestFail data -> + CachedFuzzTestFail data -> data.fuzzerInts else @@ -253,8 +269,8 @@ dispatchFuzzTest testId model = runWithDuration (\() -> fuzzTest.thunk seed model.runInfo.fuzzRuns fuzzerInts) in case expectation_ of - FuzzTestPass _ -> - ( expectation_ + FuzzTestPass data -> + ( CachedFuzzTestPass data , duration_ , getAndClearDebugLogs False ) @@ -270,7 +286,13 @@ dispatchFuzzTest testId model = |> (\() -> getAndClearDebugLogs False) ) in - ( expectation_ + ( CachedFuzzTestFail + { given = data.given + , fuzzerInts = data.fuzzerInts + , description = data.description + , reason = data.reason + , distributionReport = data.distributionReport + } , duration_ , newDebugLogs ) @@ -279,7 +301,7 @@ dispatchFuzzTest testId model = sendFuzzTestResult testId fuzzTest expectation duration debugLogs model.testReporter -sendFuzzTestResult : TestId -> FuzzTest -> FuzzTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg +sendFuzzTestResult : TestId -> FuzzTest -> CachedFuzzTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = let jsDefinitionName = @@ -290,10 +312,10 @@ sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = outcome = case expectation of - FuzzTestPass { distributionReport } -> + CachedFuzzTestPass { distributionReport } -> Passed distributionReport - FuzzTestFail { given, description, reason, distributionReport } -> + CachedFuzzTestFail { given, description, reason, distributionReport } -> Failed ( { given = given , description = description @@ -314,14 +336,11 @@ sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = testReporter.reportComplete result expectationElmCode = - if expectation == FuzzTestPass { distributionReport = NoDistribution } && not hasDebugLogs then + if expectation == CachedFuzzTestPass { distributionReport = NoDistribution } && not hasDebugLogs then Nothing else - Debug.toString expectation - -- For `rerunFailure`: - |> String.replace "" "identity" - |> Just + Just (Debug.toString expectation) in Ports.sendResult testId True jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report @@ -470,7 +489,7 @@ update msg ({ testReporter } as model) = case Dict.get fuzzTest.labels cachedTests.fuzzTests of -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. Nothing -> - Just ( FuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) + Just ( CachedFuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) cached -> cached @@ -647,10 +666,10 @@ previousRunHasFailingFuzzTest previousRun = Just ( expectation_, _ ) -> case expectation_ of - FuzzTestPass _ -> + CachedFuzzTestPass _ -> False - FuzzTestFail _ -> + CachedFuzzTestFail _ -> True ) False diff --git a/lib/Generate.js b/lib/Generate.js index 2ae82b10..ce120b7b 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -546,9 +546,9 @@ module ${previousRunModule.moduleName} exposing (previousRun) import Dict import Json.Encode import Test.Distribution exposing (DistributionReport(..)) -import Test.Runner exposing (FuzzTestExpectation(..), UnitTestExpectation(..)) +import Test.Runner exposing (UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) -import Test.Runner.Node +import Test.Runner.Node exposing (CachedFuzzTestExpectation(..)) encodeDebugLogs : List String -> Json.Encode.Value From 398b2fad0841f10080db820e5d79528186e3b552 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 16 Aug 2026 16:24:04 +0200 Subject: [PATCH 73/81] Stop using `formatLabels` from the lib --- elm/src/Test/Reporter/Console.elm | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/elm/src/Test/Reporter/Console.elm b/elm/src/Test/Reporter/Console.elm index 26ab5a34..a61c7223 100644 --- a/elm/src/Test/Reporter/Console.elm +++ b/elm/src/Test/Reporter/Console.elm @@ -7,7 +7,6 @@ import Test.Reporter.Console.Format exposing (format) import Test.Reporter.Console.Format.Color as FormatColor import Test.Reporter.Console.Format.Monochrome as FormatMonochrome import Test.Reporter.TestResults as Results exposing (Failure, Outcome(..), SummaryInfo) -import Test.Runner exposing (formatLabels) formatDuration : Float -> String @@ -36,6 +35,22 @@ pluralize singular plural count = String.join " " [ String.fromInt count, suffix ] +formatLabels : + (String -> Text) + -> (String -> Text) + -> List String + -> List Text +formatLabels formatDescription formatTest labels = + case labels of + [] -> + [] + + test :: descriptions -> + formatTest test + :: List.map formatDescription descriptions + |> List.reverse + + passedToText : List String -> String -> Text passedToText labels distributionReport = Text.concat From 9aab633ff60e0389fe63c8e8da4c0a28d3aa6c46 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 16 Aug 2026 17:01:44 +0200 Subject: [PATCH 74/81] Updates for RunnerV2 module --- elm/src/Test/Runner/Node.elm | 6 +++--- lib/Generate.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index ac3761d2..7edaf571 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -26,9 +26,9 @@ import Test exposing (Test) import Test.Distribution exposing (DistributionReport(..)) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) import Test.Reporter.TestResults exposing (Outcome(..), TestResult) -import Test.Runner exposing (FuzzTest, FuzzTestExpectation(..), Tests, UnitTest, UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..)) import Test.Runner.Ports as Ports exposing (JsMessage(..)) +import Test.RunnerV2 as Runner exposing (FuzzTest, FuzzTestExpectation(..), Tests, UnitTest, UnitTestExpectation(..)) @@ -678,7 +678,7 @@ previousRunHasFailingFuzzTest previousRun = checkTagged : a -> JsDefinitionName -> Maybe Test checkTagged value jsDefinitionName = check value - |> Maybe.map (Test.Runner.tagTest jsDefinitionName) + |> Maybe.map (Runner.tagTest jsDefinitionName) {-| Returns `Just value` if `value` is a `Test`, otherwise `Nothing`. @@ -755,7 +755,7 @@ run options possiblyTests = else let tests = - Test.Runner.toTests (Test.concat testsList) + Runner.toTests (Test.concat testsList) in Platform.worker { init = init options tests diff --git a/lib/Generate.js b/lib/Generate.js index ce120b7b..1a9fc22b 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -546,9 +546,9 @@ module ${previousRunModule.moduleName} exposing (previousRun) import Dict import Json.Encode import Test.Distribution exposing (DistributionReport(..)) -import Test.Runner exposing (UnitTestExpectation(..)) import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) import Test.Runner.Node exposing (CachedFuzzTestExpectation(..)) +import Test.RunnerV2 exposing (UnitTestExpectation(..)) encodeDebugLogs : List String -> Json.Encode.Value From bbff04d1ac21568811eacc6b0e78c4f2dde82721 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 16 Aug 2026 18:23:28 +0200 Subject: [PATCH 75/81] Updates for opaque RunnerV2 module --- elm/src/Test/Runner/Node.elm | 119 +++++++++++++++++++++++------------ lib/Generate.js | 3 +- 2 files changed, 79 insertions(+), 43 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 7edaf571..695ab500 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -97,23 +97,35 @@ type alias CachedTests = { hash : String -- As an optimization, passing unit tests without debug logs are not stored. - , unitTests : Dict (List String) ( UnitTestExpectation, DebugLogs ) + , unitTests : Dict (List String) ( CachedUnitTestExpectation, DebugLogs ) -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. , fuzzTests : Dict (List String) ( CachedFuzzTestExpectation, DebugLogs ) } -{-| Same as `FuzzTestExpectation`, but without `rerunFailure`. +{-| Non-opaque version of `UnitTestExpectation`. +-} +type CachedUnitTestExpectation + = CachedUnitTestPass + | CachedUnitTestFail + { description : String + , reason : Reason + } + + +{-| Non-opaque version of `FuzzTestExpectation`, but without `rerunFailure`. -} type CachedFuzzTestExpectation - = CachedFuzzTestPass { distributionReport : DistributionReport } + = CachedFuzzTestPass + { distributionReport : DistributionReport + } | CachedFuzzTestFail - { given : Maybe String - , fuzzerInts : List Int - , description : String + { description : String , reason : Reason , distributionReport : DistributionReport + , given : Maybe String + , fuzzerInts : List Int } @@ -162,26 +174,37 @@ dispatchUnitTest testId model = -- The unnecessary-looking tuple here ensures that `getAndClearDebugLogs` -- runs _after_ the thunk. ( ( expectation, duration ), debugLogs ) = - ( runWithDuration unitTest.thunk, getAndClearDebugLogs False ) + ( runWithDuration (\() -> Runner.runUnitTest unitTest), getAndClearDebugLogs False ) + + cachedExpectation = + case expectation of + UnitTestPass -> + CachedUnitTestPass + + UnitTestFail data -> + CachedUnitTestFail + { description = Runner.getUnitTestFailDescription data + , reason = Runner.getUnitTestFailReason data + } in - sendUnitTestResult testId unitTest expectation duration debugLogs model.testReporter + sendUnitTestResult testId unitTest cachedExpectation duration debugLogs model.testReporter -sendUnitTestResult : TestId -> UnitTest -> UnitTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg +sendUnitTestResult : TestId -> UnitTest -> CachedUnitTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = let jsDefinitionName = - unitTest.tag + Runner.getUnitTestTag unitTest hasDebugLogs = not (isEmptyDebugLogs debugLogs) outcome = case expectation of - UnitTestPass -> + CachedUnitTestPass -> Passed NoDistribution - UnitTestFail { description, reason } -> + CachedUnitTestFail { description, reason } -> if reason == TODO then Todo description @@ -194,9 +217,12 @@ sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = , NoDistribution ) + labels = + Runner.getUnitTestLabels unitTest + result : TestResult result = - { labels = unitTest.labels + { labels = labels , outcome = outcome , duration = duration , hasDebugLogs = hasDebugLogs @@ -206,13 +232,13 @@ sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = testReporter.reportComplete result expectationElmCode = - if expectation == UnitTestPass && not hasDebugLogs then + if expectation == CachedUnitTestPass && not hasDebugLogs then Nothing else Just (Debug.toString expectation) in - Ports.sendResult testId False jsDefinitionName unitTest.labels expectationElmCode debugLogs report + Ports.sendResult testId False jsDefinitionName labels expectationElmCode debugLogs report dispatchFuzzTest : TestId -> Model -> Cmd Msg @@ -224,7 +250,7 @@ dispatchFuzzTest testId model = Just fuzzTest -> let jsDefinitionName = - fuzzTest.tag + Runner.getFuzzTestTag fuzzTest fuzzerInts = if @@ -242,7 +268,7 @@ dispatchFuzzTest testId model = [] Just cachedTests -> - case Dict.get fuzzTest.labels cachedTests.fuzzTests of + case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of Nothing -> [] @@ -266,11 +292,13 @@ dispatchFuzzTest testId model = |> (\_ -> let ( expectation_, duration_ ) = - runWithDuration (\() -> fuzzTest.thunk seed model.runInfo.fuzzRuns fuzzerInts) + runWithDuration (\() -> Runner.runFuzzTest fuzzTest seed model.runInfo.fuzzRuns fuzzerInts) in case expectation_ of FuzzTestPass data -> - ( CachedFuzzTestPass data + ( CachedFuzzTestPass + { distributionReport = Runner.getFuzzTestPassDistributionReport data + } , duration_ , getAndClearDebugLogs False ) @@ -282,16 +310,16 @@ dispatchFuzzTest testId model = getAndClearDebugLogs False |> (\_ -> -- Collect debug logs from failing run. - data.rerunFailure () + Runner.rerunFuzzTestFailure data |> (\() -> getAndClearDebugLogs False) ) in ( CachedFuzzTestFail - { given = data.given - , fuzzerInts = data.fuzzerInts - , description = data.description - , reason = data.reason - , distributionReport = data.distributionReport + { description = Runner.getFuzzTestFailDescription data + , reason = Runner.getFuzzTestFailReason data + , distributionReport = Runner.getFuzzTestFailDistributionReport data + , given = Runner.getFuzzTestFailGiven data + , fuzzerInts = Runner.getFuzzTestFailFuzzerInts data } , duration_ , newDebugLogs @@ -305,7 +333,7 @@ sendFuzzTestResult : TestId -> FuzzTest -> CachedFuzzTestExpectation -> Float -> sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = let jsDefinitionName = - fuzzTest.tag + Runner.getFuzzTestTag fuzzTest hasDebugLogs = not (isEmptyDebugLogs debugLogs) @@ -324,9 +352,12 @@ sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = , distributionReport ) + labels = + Runner.getFuzzTestLabels fuzzTest + result : TestResult result = - { labels = fuzzTest.labels + { labels = labels , outcome = outcome , duration = duration , hasDebugLogs = hasDebugLogs @@ -342,7 +373,7 @@ sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = else Just (Debug.toString expectation) in - Ports.sendResult testId True jsDefinitionName fuzzTest.labels expectationElmCode debugLogs report + Ports.sendResult testId True jsDefinitionName labels expectationElmCode debugLogs report update : Msg -> Model -> ( Model, Cmd Msg ) @@ -412,7 +443,7 @@ update msg ({ testReporter } as model) = Just unitTest -> let jsDefinitionName = - unitTest.tag + Runner.getUnitTestTag unitTest hash = getHash jsDefinitionName @@ -422,10 +453,10 @@ update msg ({ testReporter } as model) = |> Maybe.andThen (\cachedTests -> if hash == cachedTests.hash then - case Dict.get unitTest.labels cachedTests.unitTests of + case Dict.get (Runner.getUnitTestLabels unitTest) cachedTests.unitTests of -- As an optimization, passing unit tests without debug logs are not stored. Nothing -> - Just ( UnitTestPass, noDebugLogs ) + Just ( CachedUnitTestPass, noDebugLogs ) cached -> cached @@ -470,7 +501,7 @@ update msg ({ testReporter } as model) = Just fuzzTest -> let jsDefinitionName = - fuzzTest.tag + Runner.getFuzzTestTag fuzzTest hash = getHash jsDefinitionName @@ -484,9 +515,9 @@ update msg ({ testReporter } as model) = && (model.runInfo.initialSeed == model.previousRun.initialSeed) -- If the fuzz tests specifies its own number of runs and the hash is the same, -- then the number of runs must be unchanged. - && (fuzzTest.runs /= Nothing || model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) + && (Runner.getFuzzTestRuns fuzzTest /= Nothing || model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) then - case Dict.get fuzzTest.labels cachedTests.fuzzTests of + case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. Nothing -> Just ( CachedFuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) @@ -531,7 +562,7 @@ init : RunnerOptions -> Tests -> Bool -> ( Model, Cmd Msg ) init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = let autoFail = - case ( tests.seenOnly, tests.seenSkip ) of + case ( Runner.getSeenOnly tests, Runner.getSeenSkip tests ) of ( False, False ) -> Nothing @@ -544,8 +575,14 @@ init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = ( True, True ) -> Just "Test.only and Test.skip were used" + unitTests = + Runner.getUnitTests tests + + fuzzTests = + Runner.getFuzzTests tests + testCount = - Array.length tests.unitTests + Array.length tests.fuzzTests + Array.length unitTests + Array.length fuzzTests testReporter = createReporter report @@ -556,7 +593,7 @@ init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = seed_ RandomSeed seed_ -> - if previousRunHasFailingFuzzTest previousRun tests.fuzzTests then + if previousRunHasFailingFuzzTest previousRun fuzzTests then previousRun.initialSeed else @@ -564,8 +601,8 @@ init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = model : Model model = - { unitTests = tests.unitTests - , fuzzTests = tests.fuzzTests + { unitTests = unitTests + , fuzzTests = fuzzTests , runInfo = { testCount = testCount , globs = globs @@ -653,14 +690,14 @@ previousRunHasFailingFuzzTest previousRun = else let jsDefinitionName = - fuzzTest.tag + Runner.getFuzzTestTag fuzzTest in case Dict.get jsDefinitionName previousRun.cachedTests of Nothing -> False Just cachedTests -> - case Dict.get fuzzTest.labels cachedTests.fuzzTests of + case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of Nothing -> False diff --git a/lib/Generate.js b/lib/Generate.js index 1a9fc22b..fc203327 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -547,8 +547,7 @@ import Dict import Json.Encode import Test.Distribution exposing (DistributionReport(..)) import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) -import Test.Runner.Node exposing (CachedFuzzTestExpectation(..)) -import Test.RunnerV2 exposing (UnitTestExpectation(..)) +import Test.Runner.Node exposing (CachedFuzzTestExpectation(..), CachedUnitTestExpectation(..)) encodeDebugLogs : List String -> Json.Encode.Value From ba39d965cc03d84642ac3f1b5bd3152add80fe9e Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 16 Aug 2026 19:36:46 +0200 Subject: [PATCH 76/81] Fix elm-review error --- elm/src/Test/Runner/Node.elm | 33 +++++++++------------------------ lib/Generate.js | 7 ++----- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 695ab500..342f7c08 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,7 +1,4 @@ -module Test.Runner.Node exposing - ( checkTagged, run, TestProgram, PreviousRun, SeedChoice(..) - , CachedFuzzTestExpectation(..) - ) +module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun, CachedUnitTestExpectation(..), CachedFuzzTestExpectation(..)) {-| @@ -11,7 +8,7 @@ module Test.Runner.Node exposing Runs a test and outputs its results to the console. Exit code is 0 if tests passed and 2 if any failed. Returns 1 if something went wrong. -@docs checkTagged, run, TestProgram, PreviousRun, SeedChoice +@docs checkTagged, run, TestProgram, PreviousRun, CachedUnitTestExpectation, CachedFuzzTestExpectation -} @@ -58,7 +55,8 @@ type alias DebugLogs = type alias RunnerOptions = - { seed : SeedChoice + { seed : Int + , seedIsUserSupplied : Bool , runs : Int , report : Report , globs : List String @@ -67,14 +65,6 @@ type alias RunnerOptions = } -type SeedChoice - = UserSuppliedSeed Int - -- Note: We might not use the random seed here; - -- we might end up using the same seed as the previous run instead - -- if that reproduces a fuzz test failure. - | RandomSeed Int - - type alias Model = { unitTests : Array UnitTest , fuzzTests : Array FuzzTest @@ -559,7 +549,7 @@ update msg ({ testReporter } as model) = init : RunnerOptions -> Tests -> Bool -> ( Model, Cmd Msg ) -init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = +init { globs, paths, runs, seed, seedIsUserSupplied, report, previousRun } tests shouldSendBegin = let autoFail = case ( Runner.getSeenOnly tests, Runner.getSeenSkip tests ) of @@ -588,16 +578,11 @@ init { globs, paths, runs, seed, report, previousRun } tests shouldSendBegin = createReporter report initialSeed = - case seed of - UserSuppliedSeed seed_ -> - seed_ + if not seedIsUserSupplied && previousRunHasFailingFuzzTest previousRun fuzzTests then + previousRun.initialSeed - RandomSeed seed_ -> - if previousRunHasFailingFuzzTest previousRun fuzzTests then - previousRun.initialSeed - - else - seed_ + else + seed model : Model model = diff --git a/lib/Generate.js b/lib/Generate.js index 3daf6729..1f3c0dc2 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -422,11 +422,8 @@ function makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) { return ` { runs = ${fuzz} , report = ${generateElmReportVariant(report)} -, seed = ${ - seed === null - ? `Test.Runner.Node.RandomSeed ${makeRandomSeed()}` - : `Test.Runner.Node.UserSuppliedSeed ${seed}` - } +, seed = ${seed === null ? makeRandomSeed() : seed} +, seedIsUserSupplied = ${seed === null ? 'False' : 'True'} , previousRun = ${previousRunModuleName.join('.')}.previousRun , globs = ${indentAllButFirstLine(' ', makeList(testFileGlobs.map(makeElmString)))} From 6b2e7049e1732c58d659a8a001424ba0ce73a4c5 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Sun, 16 Aug 2026 19:51:58 +0200 Subject: [PATCH 77/81] Fix flaky tests --- lib/Generate.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/Generate.js b/lib/Generate.js index 1f3c0dc2..ff1046dd 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -579,7 +579,12 @@ previousRun = fs.mkdirSync(path.dirname(previousRunModule.path), { recursive: true }); - fs.writeFileSync(previousRunModule.path, fileContents); + // Write to a temporary file and then rename it atomically to the actual path. + // This avoids ending up with an empty file is elm-test is killed right between + // the file is truncated and written to. The tests sometimes failed due to this. + const tempPath = previousRunModule.path + '.tmp'; + fs.writeFileSync(tempPath, fileContents); + fs.renameSync(tempPath, previousRunModule.path); } module.exports = { From 1be7e8c21e6eb09dcd36e9ae5794ffcbcee1d45e Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 18 Aug 2026 18:31:38 +0200 Subject: [PATCH 78/81] Optimize formatLabels --- elm/src/Test/Reporter/Console.elm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/elm/src/Test/Reporter/Console.elm b/elm/src/Test/Reporter/Console.elm index a61c7223..b2841e33 100644 --- a/elm/src/Test/Reporter/Console.elm +++ b/elm/src/Test/Reporter/Console.elm @@ -46,9 +46,10 @@ formatLabels formatDescription formatTest labels = [] test :: descriptions -> - formatTest test - :: List.map formatDescription descriptions - |> List.reverse + List.foldl + (\x acc -> formatDescription x :: acc) + [ formatTest test ] + descriptions passedToText : List String -> String -> Text From b212767629f742163587ce5ca5e6cc88b9dabab0 Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 18 Aug 2026 18:36:11 +0200 Subject: [PATCH 79/81] CachedFuzzTestPass DistributionReport without unnecessary record --- elm/src/Test/Runner/Node.elm | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 342f7c08..cb3eefdb 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -107,9 +107,7 @@ type CachedUnitTestExpectation {-| Non-opaque version of `FuzzTestExpectation`, but without `rerunFailure`. -} type CachedFuzzTestExpectation - = CachedFuzzTestPass - { distributionReport : DistributionReport - } + = CachedFuzzTestPass DistributionReport | CachedFuzzTestFail { description : String , reason : Reason @@ -286,9 +284,7 @@ dispatchFuzzTest testId model = in case expectation_ of FuzzTestPass data -> - ( CachedFuzzTestPass - { distributionReport = Runner.getFuzzTestPassDistributionReport data - } + ( CachedFuzzTestPass (Runner.getFuzzTestPassDistributionReport data) , duration_ , getAndClearDebugLogs False ) @@ -330,7 +326,7 @@ sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = outcome = case expectation of - CachedFuzzTestPass { distributionReport } -> + CachedFuzzTestPass distributionReport -> Passed distributionReport CachedFuzzTestFail { given, description, reason, distributionReport } -> @@ -357,7 +353,7 @@ sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = testReporter.reportComplete result expectationElmCode = - if expectation == CachedFuzzTestPass { distributionReport = NoDistribution } && not hasDebugLogs then + if expectation == CachedFuzzTestPass NoDistribution && not hasDebugLogs then Nothing else @@ -510,7 +506,7 @@ update msg ({ testReporter } as model) = case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. Nothing -> - Just ( CachedFuzzTestPass { distributionReport = NoDistribution }, noDebugLogs ) + Just ( CachedFuzzTestPass NoDistribution, noDebugLogs ) cached -> cached From 3d6a1db3e622528e8eded4264fd625165020b23f Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 18 Aug 2026 18:39:25 +0200 Subject: [PATCH 80/81] Replace `if reason == TODO then` with `case reason of` --- elm/src/Test/Runner/Node.elm | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index cb3eefdb..a0e71a22 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -193,17 +193,18 @@ sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = Passed NoDistribution CachedUnitTestFail { description, reason } -> - if reason == TODO then - Todo description - - else - Failed - ( { given = Nothing - , description = description - , reason = reason - } - , NoDistribution - ) + case reason of + TODO -> + Todo description + + _ -> + Failed + ( { given = Nothing + , description = description + , reason = reason + } + , NoDistribution + ) labels = Runner.getUnitTestLabels unitTest From ce7a550510c9fc566b8dd16de5f963f3f8cf142c Mon Sep 17 00:00:00 2001 From: Simon Lydell Date: Tue, 18 Aug 2026 18:44:39 +0200 Subject: [PATCH 81/81] Fix "your" -> "you" in README.md Co-authored-by: Jeroen Engels --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7aa3dcd..52b87e54 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ By default, the console is cleared before each run in watch mode, so you only se elm-test collects all `Debug.log` output while executing a test, and displays it all once the test in question is finished. This way elm-test can print _which_ test the logs came from. -If the function your are testing gets into an infinite loop, it means that your debug logs will never show up. Then it can be useful to have the logs print _immediately_ instead (at the loss of no longer being able to label which tests the logs came from). To avoid confusion, use [Test.only](https://package.elm-lang.org/packages/elm-explorations/test/latest/Test#only) to isolate your test, or pass `--workers 1` to run in single-threaded mode to avoid oddly mixed output: +If the function you are testing gets into an infinite loop, it means that your debug logs will never show up. Then it can be useful to have the logs print _immediately_ instead (at the loss of no longer being able to label which tests the logs came from). To avoid confusion, use [Test.only](https://package.elm-lang.org/packages/elm-explorations/test/latest/Test#only) to isolate your test, or pass `--workers 1` to run in single-threaded mode to avoid oddly mixed output: elm-test --unbuffered-logs --workers 1