diff --git a/README.md b/README.md index 8c2146c3..52b87e54 100644 --- a/README.md +++ b/README.md @@ -156,12 +156,30 @@ Start the runner in watch mode. Your tests will automatically rerun whenever you elm-test --watch +### --no-clear-console + +By default, the console is cleared before each run in watch mode, so you only see the latest information. If you don’t like this, turn it off with `--no-clear-console`. + + elm-test --watch --no-clear-console + +### --unbuffered-logs + +elm-test collects all `Debug.log` output while executing a test, and displays it all once the test in question is finished. This way elm-test can print _which_ test the logs came from. + +If the function you are testing gets into an infinite loop, it means that your debug logs will never show up. Then it can be useful to have the logs print _immediately_ instead (at the loss of no longer being able to label which tests the logs came from). To avoid confusion, use [Test.only](https://package.elm-lang.org/packages/elm-explorations/test/latest/Test#only) to isolate your test, or pass `--workers 1` to run in single-threaded mode to avoid oddly mixed output: + + elm-test --unbuffered-logs --workers 1 + +For _failing_ fuzz tests, elm-test only prints `Debug.log` output from the run of the fuzz test that produced the failure, which is usually what you want to debug. (Earlier, passing runs of the function with different input is just noise). For _passing_ fuzz tests, elm-test _ignores_ your `Debug.log` calls (and instead displays a note about this). Let’s imagine you are debugging a failing fuzz test. After a while it finally passes. There is no longer a failing run, so which one should we pick logs from? All of them? But would you really like to see the screen fill with 100+ repetitions of your logs at that point? Probably not. But if you actually _do_ want to show logs from all runs, you can use `--unbuffered-logs` for this use case, too. Also remember that you can make the test fail from anywhere using `Debug.todo` – that’s also a way to make logs appear! + ### --seed Run with a specific fuzzer seed, rather than a randomly generated seed. This allows reproducing a failing fuzz-test. The command needed to reproduce (including the `--seed` flag) is printed after each test run. Copy, paste and run it! elm-test --seed 336948560956134 +On top of that, if you run elm-test without the `--seed` flag, elm-test will automatically use the same seed as the last run if there was a fuzz test failure, letting you reproduce errors without doing anything. It even tries to fast-forward you through the fuzzing. So if it took some time for the fuzzer to find the problem the first time, the next run should be instant. + ### --fuzz Define how many times each fuzz-test should run. Defaults to `100`. @@ -173,16 +191,18 @@ Define how many times each fuzz-test should run. Defaults to `100`. ### --workers -Choose how many workers elm-test should use to run tests in parallel. Defaults to the number of “logical CPU cores” of the machine you run the tests on. +Choose how many workers elm-test should use to run fuzz tests in parallel. Defaults to the number of “logical CPU cores” of the machine you run the tests on. elm-test --workers 4 -Your computer might say that it has 12 logical CPU cores. Then dividing up the tests between 12 parallel workers is the theoretical optimum for running the tests as quickly as possible. But in practice your tests might run faster with just 4 workers in parallel due to overhead. Play around with it and see what is the fastest for your test suite on your computer! +Your computer might say that it has 12 logical CPU cores. Then dividing up the fuzz tests between 12 parallel workers is the theoretical optimum for running the tests as quickly as possible. But in practice your tests might run faster with just 4 workers in parallel due to overhead. Play around with it and see what is the fastest for your test suite on your computer! To see the number of logical CPU cores on your machine, run `node -p "os.cpus().length"` (it’s also shown in `elm-test --help`). If you pass `--workers 1`, elm-test won’t even start a new thread for running the tests in – it’ll do everything in the main thread (single-threaded mode). +Currently, elm-test always executes unit tests on the main thread, and only uses separate threads for fuzz tests. Unit tests tend to execute so fast that the overhead of threads isn’t worth it. But fuzz tests often run long enough to benefit from parallelization. + ### --report Specify which format to use for reporting test results. Valid options are: diff --git a/elm/elm.json b/elm/elm.json index 08e5084d..b83f716f 100644 --- a/elm/elm.json +++ b/elm/elm.json @@ -9,12 +9,12 @@ "elm/core": "1.0.5", "elm/json": "1.1.3", "elm/random": "1.0.0", - "elm/time": "1.0.0", "elm-explorations/test": "2.2.1" }, "indirect": { "elm/bytes": "1.0.8", "elm/html": "1.0.0", + "elm/time": "1.0.0", "elm/virtual-dom": "1.0.3" } }, diff --git a/elm/src/Test/Reporter/Console.elm b/elm/src/Test/Reporter/Console.elm index ad8aed30..b2841e33 100644 --- a/elm/src/Test/Reporter/Console.elm +++ b/elm/src/Test/Reporter/Console.elm @@ -7,7 +7,6 @@ import Test.Reporter.Console.Format exposing (format) import Test.Reporter.Console.Format.Color as FormatColor import Test.Reporter.Console.Format.Monochrome as FormatMonochrome import Test.Reporter.TestResults as Results exposing (Failure, Outcome(..), SummaryInfo) -import Test.Runner exposing (formatLabels) formatDuration : Float -> String @@ -36,6 +35,23 @@ pluralize singular plural count = String.join " " [ String.fromInt count, suffix ] +formatLabels : + (String -> Text) + -> (String -> Text) + -> List String + -> List Text +formatLabels formatDescription formatTest labels = + case labels of + [] -> + [] + + test :: descriptions -> + List.foldl + (\x acc -> formatDescription x :: acc) + [ formatTest test ] + descriptions + + passedToText : List String -> String -> Text passedToText labels distributionReport = Text.concat @@ -147,7 +163,7 @@ getStatus outcome = reportComplete : UseColor -> Results.TestResult -> Value -reportComplete useColor { labels, outcome } = +reportComplete useColor { labels, outcome, hasDebugLogs } = Encode.object <| ( "type", Encode.string "complete" ) :: ( "status", Encode.string (getStatus outcome) ) @@ -156,10 +172,18 @@ reportComplete useColor { labels, outcome } = -- No failures of any kind. case distributionReportToString distributionReport of Nothing -> - [] + if hasDebugLogs then + [ ( "message" + , passedLabelsToText labels + |> textToValue useColor + ) + ] + + else + [] Just report -> - [ ( "distributionReport" + [ ( "message" , report |> passedToText labels |> textToValue useColor diff --git a/elm/src/Test/Reporter/JUnit.elm b/elm/src/Test/Reporter/JUnit.elm index 234f2d68..61e20f9f 100644 --- a/elm/src/Test/Reporter/JUnit.elm +++ b/elm/src/Test/Reporter/JUnit.elm @@ -84,9 +84,9 @@ formatClassAndName labels = ( "", "" ) -encodeDuration : Int -> Value +encodeDuration : Float -> Value encodeDuration time = - (toFloat time / 1000) + (time / 1000) |> String.fromFloat |> Encode.string @@ -119,6 +119,7 @@ encodeExtraFailure _ = } , NoDistribution ) + , hasDebugLogs = False } diff --git a/elm/src/Test/Reporter/Json.elm b/elm/src/Test/Reporter/Json.elm index 643076d9..2b200ea2 100644 --- a/elm/src/Test/Reporter/Json.elm +++ b/elm/src/Test/Reporter/Json.elm @@ -28,7 +28,11 @@ reportComplete { duration, labels, outcome } = , ( "labels", encodeLabels labels ) , ( "failures", Encode.list identity (encodeFailures outcome) ) , ( "distributionReports", Encode.list identity (encodeDistributionReports outcome) ) - , ( "duration", Encode.string <| String.fromInt duration ) + + -- Keep the "duration" field Int for backwards compatibility, + -- and also expose the new Float field for more precision. + , ( "duration", Encode.string <| String.fromInt (round duration) ) + , ( "durationFloat", Encode.string <| String.fromFloat duration ) ] diff --git a/elm/src/Test/Reporter/TestResults.elm b/elm/src/Test/Reporter/TestResults.elm index 53390e36..e08783f0 100644 --- a/elm/src/Test/Reporter/TestResults.elm +++ b/elm/src/Test/Reporter/TestResults.elm @@ -3,13 +3,9 @@ module Test.Reporter.TestResults exposing , Outcome(..) , SummaryInfo , TestResult - , isFailure - , outcomeFromExpectations ) -import Expect exposing (Expectation) import Test.Distribution exposing (DistributionReport) -import Test.Runner import Test.Runner.Failure exposing (Reason) @@ -22,7 +18,8 @@ type Outcome type alias TestResult = { labels : List String , outcome : Outcome - , duration : Int -- in milliseconds + , duration : Float -- in milliseconds + , hasDebugLogs : Bool } @@ -40,38 +37,3 @@ type alias Failure = , description : String , reason : Reason } - - -isFailure : Outcome -> Bool -isFailure outcome = - case outcome of - Failed _ -> - True - - _ -> - False - - -outcomeFromExpectations : List Expectation -> Outcome -outcomeFromExpectations expectations = - case expectations of - -- The type of test runner functions says that they return `List Expectation`, - -- but in practice they only ever return lists with exactly one item: - -- https://github.com/elm-explorations/test/pull/244 - -- That PR was reverted because it unfortunately was a breaking change for the package: - -- https://github.com/elm-explorations/test/commit/11f70d5fc0b6fdc88d7a34ea1d10f56969890493 - -- But to keep things simpler here, we only support exactly one expectation. - [ expectation ] -> - case Test.Runner.getFailureReason expectation of - Nothing -> - Passed (Test.Runner.getDistributionReport expectation) - - Just failure -> - if Test.Runner.isTodo expectation then - Todo failure.description - - else - Failed ( failure, Test.Runner.getDistributionReport expectation ) - - _ -> - Debug.todo ("A test somehow did not return exactly 1 expectation, it returned " ++ String.fromInt (List.length expectations) ++ "!") diff --git a/elm/src/Test/Runner/JsMessage.elm b/elm/src/Test/Runner/JsMessage.elm deleted file mode 100644 index a6a5c5bd..00000000 --- a/elm/src/Test/Runner/JsMessage.elm +++ /dev/null @@ -1,33 +0,0 @@ -module Test.Runner.JsMessage exposing (JsMessage(..), decoder) - -import Json.Decode as Decode exposing (Decoder) - - -type JsMessage - = Summary Float Int (List ( List String, String )) - - -decoder : Decoder JsMessage -decoder = - Decode.field "type" Decode.string - |> Decode.andThen decodeMessageFromType - - -decodeMessageFromType : String -> Decoder JsMessage -decodeMessageFromType messageType = - case messageType of - "SUMMARY" -> - Decode.map3 Summary - (Decode.field "duration" Decode.float) - (Decode.field "failures" Decode.int) - (Decode.field "todos" (Decode.list todoDecoder)) - - _ -> - Decode.fail ("Unrecognized message type: " ++ messageType) - - -todoDecoder : Decoder ( List String, String ) -todoDecoder = - Decode.map2 (\a b -> ( a, b )) - (Decode.field "labels" (Decode.list Decode.string)) - (Decode.field "todo" Decode.string) diff --git a/elm/src/Test/Runner/Node.elm b/elm/src/Test/Runner/Node.elm index 679d7e35..a0e71a22 100644 --- a/elm/src/Test/Runner/Node.elm +++ b/elm/src/Test/Runner/Node.elm @@ -1,4 +1,4 @@ -port module Test.Runner.Node exposing (check, run, TestProgram) +module Test.Runner.Node exposing (checkTagged, run, TestProgram, PreviousRun, CachedUnitTestExpectation(..), CachedFuzzTestExpectation(..)) {-| @@ -8,10 +8,11 @@ port module Test.Runner.Node exposing (check, run, TestProgram) Runs a test and outputs its results to the console. Exit code is 0 if tests passed and 2 if any failed. Returns 1 if something went wrong. -@docs check, run, TestProgram +@docs checkTagged, run, TestProgram, PreviousRun, CachedUnitTestExpectation, CachedFuzzTestExpectation -} +import Array exposing (Array) import Dict exposing (Dict) import Json.Decode as Decode import Json.Encode as Encode @@ -19,96 +20,361 @@ import Platform import Random import Task import Test exposing (Test) +import Test.Distribution exposing (DistributionReport(..)) import Test.Reporter.Reporter exposing (Report, RunInfo, TestReporter, createReporter) -import Test.Reporter.TestResults exposing (Outcome, TestResult, isFailure, outcomeFromExpectations) -import Test.Runner exposing (Runner, SeededRunners(..)) -import Test.Runner.JsMessage as JsMessage exposing (JsMessage(..)) -import Time exposing (Posix) +import Test.Reporter.TestResults exposing (Outcome(..), TestResult) +import Test.Runner.Failure exposing (Reason(..)) +import Test.Runner.Ports as Ports exposing (JsMessage(..)) +import Test.RunnerV2 as Runner exposing (FuzzTest, FuzzTestExpectation(..), Tests, UnitTest, UnitTestExpectation(..)) -- TYPES +{-| A `TestId` is just an index into an `Array` of tests. +-} type alias TestId = Int -type alias InitArgs = - { initialSeed : Int - , processes : Int - , globs : List String - , paths : List String - , fuzzRuns : Int - , runners : SeededRunners - , report : Report - } +{-| The compiled JavaScript name of an exposed value, +such as `$user$project$Tests$suite`. +-} +type alias JsDefinitionName = + String + + +{-| Collected `Debug.log`s during a test (or during initialization before running any tests). +There are stored as `Decode.Value` instead of `List String` as an optimization. They are +collected in JavaScript code, given to Elm for a short while, and then sent through a port. +So going to from a JS array, to an Elm list, back to a JS array is pretty wasteful. +-} +type alias DebugLogs = + Decode.Value type alias RunnerOptions = { seed : Int + , seedIsUserSupplied : Bool , runs : Int , report : Report , globs : List String , paths : List String - , processes : Int + , previousRun : PreviousRun } type alias Model = - { available : Dict TestId Runner + { unitTests : Array UnitTest + , fuzzTests : Array FuzzTest , runInfo : RunInfo , testReporter : TestReporter - , results : List ( TestId, TestResult ) - , processes : Int - , nextTestToRun : TestId , autoFail : Maybe String + , previousRun : PreviousRun + , cacheTrawl : CacheTrawl + } + + +type alias PreviousRun = + { fuzzRuns : Int + , initialSeed : Int + , cachedTests : Dict JsDefinitionName CachedTests } +type alias CachedTests = + { hash : String + + -- As an optimization, passing unit tests without debug logs are not stored. + , unitTests : Dict (List String) ( CachedUnitTestExpectation, DebugLogs ) + + -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. + , fuzzTests : Dict (List String) ( CachedFuzzTestExpectation, DebugLogs ) + } + + +{-| Non-opaque version of `UnitTestExpectation`. +-} +type CachedUnitTestExpectation + = CachedUnitTestPass + | CachedUnitTestFail + { description : String + , reason : Reason + } + + +{-| Non-opaque version of `FuzzTestExpectation`, but without `rerunFailure`. +-} +type CachedFuzzTestExpectation + = CachedFuzzTestPass DistributionReport + | CachedFuzzTestFail + { description : String + , reason : Reason + , distributionReport : DistributionReport + , given : Maybe String + , fuzzerInts : List Int + } + + +type CacheTrawl + = NotTrawling + | TrawlingUnitTests + { current : TestId + , unitTests : List TestId + } + | TrawlingFuzzTests + { current : TestId + , unitTests : List TestId + , fuzzTests : List TestId + } + + {-| A program which will run tests and report their results. -} type alias TestProgram = - Platform.Program Int Model Msg + Program Bool Model Msg type Msg - = Receive Decode.Value - | Dispatch Posix - | Complete (List String) Outcome Posix Posix + = Receive (Result Decode.Error JsMessage) + | Trawl -{-| The port names are prefixed to reduce the likelihood of the project -having a port with the same name, which is a compile error. --} -port elmTestPort__send : Decode.Value -> Cmd msg +noDebugLogs : DebugLogs +noDebugLogs = + Encode.list never [] -port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg +isEmptyDebugLogs : DebugLogs -> Bool +isEmptyDebugLogs debugLogs = + Decode.decodeValue (Decode.field "length" Decode.int) debugLogs == Ok 0 -dispatch : Model -> Posix -> Cmd Msg -dispatch model startTime = - case Dict.get model.nextTestToRun model.available of +dispatchUnitTest : TestId -> Model -> Cmd Msg +dispatchUnitTest testId model = + case Array.get testId model.unitTests of Nothing -> - -- We're finished! Nothing left to run. - sendResults True model.testReporter model.results + Ports.sendError ("Unit test not found: " ++ String.fromInt testId) - Just config -> + Just unitTest -> let - outcome = - outcomeFromExpectations (config.run ()) + -- The unnecessary-looking tuple here ensures that `getAndClearDebugLogs` + -- runs _after_ the thunk. + ( ( expectation, duration ), debugLogs ) = + ( runWithDuration (\() -> Runner.runUnitTest unitTest), getAndClearDebugLogs False ) + + cachedExpectation = + case expectation of + UnitTestPass -> + CachedUnitTestPass + + UnitTestFail data -> + CachedUnitTestFail + { description = Runner.getUnitTestFailDescription data + , reason = Runner.getUnitTestFailReason data + } in - Time.now - |> Task.perform (Complete config.labels outcome startTime) + sendUnitTestResult testId unitTest cachedExpectation duration debugLogs model.testReporter + + +sendUnitTestResult : TestId -> UnitTest -> CachedUnitTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg +sendUnitTestResult testId unitTest expectation duration debugLogs testReporter = + let + jsDefinitionName = + Runner.getUnitTestTag unitTest + + hasDebugLogs = + not (isEmptyDebugLogs debugLogs) + + outcome = + case expectation of + CachedUnitTestPass -> + Passed NoDistribution + + CachedUnitTestFail { description, reason } -> + case reason of + TODO -> + Todo description + + _ -> + Failed + ( { given = Nothing + , description = description + , reason = reason + } + , NoDistribution + ) + + labels = + Runner.getUnitTestLabels unitTest + + result : TestResult + result = + { labels = labels + , outcome = outcome + , duration = duration + , hasDebugLogs = hasDebugLogs + } + + report = + testReporter.reportComplete result + + expectationElmCode = + if expectation == CachedUnitTestPass && not hasDebugLogs then + Nothing + + else + Just (Debug.toString expectation) + in + Ports.sendResult testId False jsDefinitionName labels expectationElmCode debugLogs report + + +dispatchFuzzTest : TestId -> Model -> Cmd Msg +dispatchFuzzTest testId model = + case Array.get testId model.fuzzTests of + Nothing -> + Ports.sendError ("Fuzz test not found: " ++ String.fromInt testId) + + Just fuzzTest -> + let + jsDefinitionName = + Runner.getFuzzTestTag fuzzTest + + fuzzerInts = + if + -- In `init` we use the same seed as the previous run if there was a failing fuzz test. + -- If the user has explicitly passed a different seed, don’t try to reproduce the previous + -- failure. They are clearly trying to run something else. + (model.runInfo.initialSeed == model.previousRun.initialSeed) + -- The number of fuzz runs must be the same (or more) as the previous run – otherwise + -- the user has explicitly passed fewer, and we can’t know if the previous failure + -- would hit or not. Since the seed is still the same, there’s still a chance it will. + && (model.runInfo.fuzzRuns >= model.previousRun.fuzzRuns) + then + case Dict.get jsDefinitionName model.previousRun.cachedTests of + Nothing -> + [] + + Just cachedTests -> + case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of + Nothing -> + [] + + Just ( expectation_, _ ) -> + case expectation_ of + CachedFuzzTestPass _ -> + [] + + CachedFuzzTestFail data -> + data.fuzzerInts + + else + [] + + seed = + Random.initialSeed model.runInfo.initialSeed + + ( expectation, duration, debugLogs ) = + -- Pause debug logs. + getAndClearDebugLogs True + |> (\_ -> + let + ( expectation_, duration_ ) = + runWithDuration (\() -> Runner.runFuzzTest fuzzTest seed model.runInfo.fuzzRuns fuzzerInts) + in + case expectation_ of + FuzzTestPass data -> + ( CachedFuzzTestPass (Runner.getFuzzTestPassDistributionReport data) + , duration_ + , getAndClearDebugLogs False + ) + + FuzzTestFail data -> + let + newDebugLogs = + -- Unpause debug logs. + getAndClearDebugLogs False + |> (\_ -> + -- Collect debug logs from failing run. + Runner.rerunFuzzTestFailure data + |> (\() -> getAndClearDebugLogs False) + ) + in + ( CachedFuzzTestFail + { description = Runner.getFuzzTestFailDescription data + , reason = Runner.getFuzzTestFailReason data + , distributionReport = Runner.getFuzzTestFailDistributionReport data + , given = Runner.getFuzzTestFailGiven data + , fuzzerInts = Runner.getFuzzTestFailFuzzerInts data + } + , duration_ + , newDebugLogs + ) + ) + in + sendFuzzTestResult testId fuzzTest expectation duration debugLogs model.testReporter + + +sendFuzzTestResult : TestId -> FuzzTest -> CachedFuzzTestExpectation -> Float -> DebugLogs -> TestReporter -> Cmd Msg +sendFuzzTestResult testId fuzzTest expectation duration debugLogs testReporter = + let + jsDefinitionName = + Runner.getFuzzTestTag fuzzTest + + hasDebugLogs = + not (isEmptyDebugLogs debugLogs) + + outcome = + case expectation of + CachedFuzzTestPass distributionReport -> + Passed distributionReport + + CachedFuzzTestFail { given, description, reason, distributionReport } -> + Failed + ( { given = given + , description = description + , reason = reason + } + , distributionReport + ) + + labels = + Runner.getFuzzTestLabels fuzzTest + + result : TestResult + result = + { labels = labels + , outcome = outcome + , duration = duration + , hasDebugLogs = hasDebugLogs + } + + report = + testReporter.reportComplete result + + expectationElmCode = + if expectation == CachedFuzzTestPass NoDistribution && not hasDebugLogs then + Nothing + + else + Just (Debug.toString expectation) + in + Ports.sendResult testId True jsDefinitionName labels expectationElmCode debugLogs report update : Msg -> Model -> ( Model, Cmd Msg ) update msg ({ testReporter } as model) = case msg of - Receive val -> - case Decode.decodeValue JsMessage.decoder val of - Ok (Summary duration failed todos) -> + Receive (Ok jsMessage) -> + case jsMessage of + RunUnitTest testId -> + ( model, dispatchUnitTest testId model ) + + RunFuzzTest testId -> + ( model, dispatchFuzzTest testId model ) + + Summary duration failed todos -> let testCount = model.runInfo.testCount @@ -135,183 +401,244 @@ update msg ({ testReporter } as model) = 3 cmd = - Encode.object - [ ( "type", Encode.string "SUMMARY" ) - , ( "exitCode", Encode.int exitCode ) - , ( "message", summary ) - ] - |> elmTestPort__send + Ports.sendSummary exitCode summary in ( model, cmd ) - Err err -> - let - cmd = - Encode.object - [ ( "type", Encode.string "ERROR" ) - , ( "message", Encode.string (Decode.errorToString err) ) - ] - |> elmTestPort__send - in - ( model, cmd ) - - Dispatch startTime -> - ( model, dispatch model startTime ) - - Complete labels outcome startTime endTime -> - let - duration = - Time.posixToMillis endTime - Time.posixToMillis startTime - - results = - ( model.nextTestToRun - , { labels = labels, outcome = outcome, duration = duration } - ) - :: model.results - - nextTestToRun = - model.nextTestToRun + model.processes - - isFinished = - nextTestToRun >= model.runInfo.testCount - in - if isFinished || isFailure outcome then - let - cmd = - sendResults isFinished testReporter results - in - if isFinished then - -- Don't bother updating the model, since we're done - ( model, cmd ) - - else - -- Clear out the results, now that we've flushed them. - ( { model | nextTestToRun = nextTestToRun, results = [] } - , Cmd.batch - [ cmd - , Task.perform Dispatch Time.now - ] - ) - - else - ( { model | nextTestToRun = nextTestToRun, results = results } - , Task.perform Dispatch Time.now - ) - - -sendResults : Bool -> TestReporter -> List ( TestId, TestResult ) -> Cmd msg -sendResults isFinished testReporter results = + Receive (Err err) -> + ( model, Ports.sendError (Decode.errorToString err) ) + + Trawl -> + case model.cacheTrawl of + NotTrawling -> + ( model, Cmd.none ) + + TrawlingUnitTests data -> + case Array.get data.current model.unitTests of + Nothing -> + ( { model + | cacheTrawl = + TrawlingFuzzTests + { current = 0 + , unitTests = data.unitTests + , fuzzTests = [] + } + } + , trawlNext + ) + + Just unitTest -> + let + jsDefinitionName = + Runner.getUnitTestTag unitTest + + hash = + getHash jsDefinitionName + + maybeCached = + Dict.get jsDefinitionName model.previousRun.cachedTests + |> Maybe.andThen + (\cachedTests -> + if hash == cachedTests.hash then + case Dict.get (Runner.getUnitTestLabels unitTest) cachedTests.unitTests of + -- As an optimization, passing unit tests without debug logs are not stored. + Nothing -> + Just ( CachedUnitTestPass, noDebugLogs ) + + cached -> + cached + + else + Nothing + ) + in + case maybeCached of + Nothing -> + ( { model + | cacheTrawl = + TrawlingUnitTests + { current = data.current + 1 + , unitTests = data.current :: data.unitTests + } + } + , trawlNext + ) + + Just ( expectation, debugLogs ) -> + ( { model + | cacheTrawl = + TrawlingUnitTests + { current = data.current + 1 + , unitTests = data.unitTests + } + } + , Cmd.batch + [ trawlNext + , sendUnitTestResult data.current unitTest expectation 0 debugLogs model.testReporter + ] + ) + + TrawlingFuzzTests data -> + case Array.get data.current model.fuzzTests of + Nothing -> + ( { model | cacheTrawl = NotTrawling } + , Ports.sendReady (List.reverse data.unitTests) (List.reverse data.fuzzTests) + ) + + Just fuzzTest -> + let + jsDefinitionName = + Runner.getFuzzTestTag fuzzTest + + hash = + getHash jsDefinitionName + + maybeCached = + Dict.get jsDefinitionName model.previousRun.cachedTests + |> Maybe.andThen + (\cachedTests -> + if + (hash == cachedTests.hash) + && (model.runInfo.initialSeed == model.previousRun.initialSeed) + -- If the fuzz tests specifies its own number of runs and the hash is the same, + -- then the number of runs must be unchanged. + && (Runner.getFuzzTestRuns fuzzTest /= Nothing || model.runInfo.fuzzRuns <= model.previousRun.fuzzRuns) + then + case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of + -- As an optimization, passing fuzz tests without debug logs and distribution report are not stored. + Nothing -> + Just ( CachedFuzzTestPass NoDistribution, noDebugLogs ) + + cached -> + cached + + else + Nothing + ) + in + case maybeCached of + Nothing -> + ( { model + | cacheTrawl = + TrawlingFuzzTests + { current = data.current + 1 + , unitTests = data.unitTests + , fuzzTests = data.current :: data.fuzzTests + } + } + , trawlNext + ) + + Just ( expectation, debugLogs ) -> + ( { model + | cacheTrawl = + TrawlingFuzzTests + { current = data.current + 1 + , unitTests = data.unitTests + , fuzzTests = data.fuzzTests + } + } + , Cmd.batch + [ trawlNext + , sendFuzzTestResult data.current fuzzTest expectation 0 debugLogs model.testReporter + ] + ) + + +init : RunnerOptions -> Tests -> Bool -> ( Model, Cmd Msg ) +init { globs, paths, runs, seed, seedIsUserSupplied, report, previousRun } tests shouldSendBegin = let - typeStr = - if isFinished then - "FINISHED" + autoFail = + case ( Runner.getSeenOnly tests, Runner.getSeenSkip tests ) of + ( False, False ) -> + Nothing - else - "RESULTS" + ( True, False ) -> + Just "Test.only was used" - addToKeyValues ( testId, result ) list = - -- These are coming in in reverse order. Doing a foldl with :: - -- means we reverse the list again, while also doing the conversion! - ( String.fromInt testId, testReporter.reportComplete result ) :: list - in - Encode.object - [ ( "type", Encode.string typeStr ) - , ( "results" - , results - |> List.foldl addToKeyValues [] - |> Encode.object - ) - ] - |> elmTestPort__send - - -sendBegin : Model -> Cmd msg -sendBegin model = - let - baseFields = - [ ( "type", Encode.string "BEGIN" ) - , ( "testCount", Encode.int model.runInfo.testCount ) - ] - - extraFields = - case model.testReporter.reportBegin model.runInfo of - Just report -> - [ ( "message", report ) ] + ( False, True ) -> + Just "Test.skip was used" - Nothing -> - [] - in - Encode.object (baseFields ++ extraFields) - |> elmTestPort__send + ( True, True ) -> + Just "Test.only and Test.skip were used" + unitTests = + Runner.getUnitTests tests -init : InitArgs -> Int -> ( Model, Cmd Msg ) -init { processes, globs, paths, fuzzRuns, initialSeed, report, runners } index = - let - { indexedRunners, autoFail } = - case runners of - Plain runnerList -> - { indexedRunners = List.indexedMap (\a b -> ( a, b )) runnerList - , autoFail = Nothing - } - - Only runnerList -> - { indexedRunners = List.indexedMap (\a b -> ( a, b )) runnerList - , autoFail = Just "Test.only was used" - } - - Skipping runnerList -> - { indexedRunners = List.indexedMap (\a b -> ( a, b )) runnerList - , autoFail = Just "Test.skip was used" - } - - Invalid str -> - { indexedRunners = [] - , autoFail = Just str - } + fuzzTests = + Runner.getFuzzTests tests testCount = - List.length indexedRunners + Array.length unitTests + Array.length fuzzTests testReporter = createReporter report + initialSeed = + if not seedIsUserSupplied && previousRunHasFailingFuzzTest previousRun fuzzTests then + previousRun.initialSeed + + else + seed + + model : Model model = - { available = Dict.fromList indexedRunners + { unitTests = unitTests + , fuzzTests = fuzzTests , runInfo = { testCount = testCount , globs = globs , paths = paths - , fuzzRuns = fuzzRuns + , fuzzRuns = runs , initialSeed = initialSeed } - , processes = processes - , nextTestToRun = index - , results = [] , testReporter = testReporter , autoFail = autoFail + , previousRun = previousRun + , cacheTrawl = + if shouldSendBegin then + TrawlingUnitTests + { current = 0 + , unitTests = [] + } + + else + NotTrawling } - cmd = - Task.perform Dispatch Time.now + -- In the main thread, we log these. + -- In workers, we just clear them and ignore them – + -- they are identical to the main thread. + debugLogs = + getAndClearDebugLogs False in ( model - , Cmd.batch - [ cmd - , if index == 0 then - sendBegin model - - else - Cmd.none - ] + , if shouldSendBegin then + Cmd.batch + [ Ports.sendBegin + initialSeed + debugLogs + (model.testReporter.reportBegin model.runInfo) + , trawlNext + ] + + else + Cmd.none ) -failInit : String -> Report -> Int -> ( Model, Cmd Msg ) +trawlNext : Cmd Msg +trawlNext = + Task.perform (\() -> Trawl) (Task.succeed ()) + + +failInit : String -> Report -> Bool -> ( Model, Cmd Msg ) failInit message report _ = let + model : Model model = - { available = Dict.empty + { unitTests = Array.empty + , fuzzTests = Array.empty , runInfo = { testCount = 0 , globs = [] @@ -319,46 +646,110 @@ failInit message report _ = , fuzzRuns = 0 , initialSeed = 0 } - , processes = 0 - , nextTestToRun = 0 - , results = [] , testReporter = createReporter report , autoFail = Nothing + , previousRun = + { fuzzRuns = 0 + , initialSeed = 0 + , cachedTests = Dict.empty + } + , cacheTrawl = NotTrawling } cmd = - Encode.object - [ ( "type", Encode.string "SUMMARY" ) - , ( "exitCode", Encode.int 1 ) - , ( "message", Encode.string message ) - ] - |> elmTestPort__send + Ports.sendSummary 1 (Encode.string message) in ( model, cmd ) -{-| The implementation of this function will be replaced in the generated JS -with a version that returns `Just value` if `value` is a `Test`, otherwise `Nothing`. +previousRunHasFailingFuzzTest : PreviousRun -> Array FuzzTest -> Bool +previousRunHasFailingFuzzTest previousRun = + Array.foldl + (\fuzzTest hasFailingFuzzTest -> + if hasFailingFuzzTest then + hasFailingFuzzTest + + else + let + jsDefinitionName = + Runner.getFuzzTestTag fuzzTest + in + case Dict.get jsDefinitionName previousRun.cachedTests of + Nothing -> + False + + Just cachedTests -> + case Dict.get (Runner.getFuzzTestLabels fuzzTest) cachedTests.fuzzTests of + Nothing -> + False + + Just ( expectation_, _ ) -> + case expectation_ of + CachedFuzzTestPass _ -> + False + + CachedFuzzTestFail _ -> + True + ) + False + -If you rename or change this function you also need to update the regex that looks for it. +checkTagged : a -> JsDefinitionName -> Maybe Test +checkTagged value jsDefinitionName = + check value + |> Maybe.map (Runner.tagTest jsDefinitionName) + +{-| Returns `Just value` if `value` is a `Test`, otherwise `Nothing`. -} check : a -> Maybe Test check = - checkHelperReplaceMe___ + placeholderReplaceMe___ "check" + + +{-| Returns all debug logs created since the beginning, +or last time this function was called. + +The `Bool` is set to `True` for fuzz tests, and pauses +`Debug.log` - logging is not supported for passing fuzz +tests. If the fuzz test fails, the failing run is run +again and that time we do collect logs. + +-} +getAndClearDebugLogs : Bool -> DebugLogs +getAndClearDebugLogs = + placeholderReplaceMe___ "getAndClearDebugLogs" + + +{-| Takes a `jsDefinitionName` and returns its hash. +-} +getHash : JsDefinitionName -> String +getHash = + placeholderReplaceMe___ "getHash" + + +runWithDuration : (() -> a) -> ( a, Float ) +runWithDuration = + placeholderReplaceMe___ "runWithDuration" -checkHelperReplaceMe___ : a -> b -checkHelperReplaceMe___ _ = - Debug.todo "The regex for replacing this Debug.todo with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n" +{-| The implementation of functions calling this one will be replaced in the generated JS +with versions that do something not normally possible in Elm. + +If you rename or change this function, or any function that calls it, you also need to update the regexes that looks for it. + +-} +placeholderReplaceMe___ : String -> a +placeholderReplaceMe___ name = + Debug.todo ("The regex for replacing this Debug.todo for '" ++ name ++ "' with some real code must have failed since you see this message!\n\nPlease report this bug: https://github.com/rtfeldman/node-test-runner/issues/new\n") {-| Run the tests. -} -run : RunnerOptions -> List ( String, List (Maybe Test) ) -> Program Int Model Msg -run { runs, seed, report, globs, paths, processes } possiblyTests = +run : RunnerOptions -> List ( String, List (Maybe Test) ) -> TestProgram +run options possiblyTests = let - tests = + testsList = possiblyTests |> List.filterMap (\( moduleName, maybeModuleTests ) -> @@ -373,33 +764,22 @@ run { runs, seed, report, globs, paths, processes } possiblyTests = Just (Test.describe moduleName moduleTests) ) in - if List.isEmpty tests then + if List.isEmpty testsList then Platform.worker - { init = failInit (noTestsFoundError globs) report + { init = failInit (noTestsFoundError options.globs) options.report , update = \_ model -> ( model, Cmd.none ) , subscriptions = \_ -> Sub.none } else let - runners = - Test.Runner.fromTest runs (Random.initialSeed seed) (Test.concat tests) - - wrappedInit = - init - { initialSeed = seed - , processes = processes - , globs = globs - , paths = paths - , fuzzRuns = runs - , runners = runners - , report = report - } + tests = + Runner.toTests (Test.concat testsList) in Platform.worker - { init = wrappedInit + { init = init options tests , update = update - , subscriptions = \_ -> elmTestPort__receive Receive + , subscriptions = \_ -> Ports.receive Receive } diff --git a/elm/src/Test/Runner/Ports.elm b/elm/src/Test/Runner/Ports.elm new file mode 100644 index 00000000..6aa3c029 --- /dev/null +++ b/elm/src/Test/Runner/Ports.elm @@ -0,0 +1,150 @@ +port module Test.Runner.Ports exposing (JsMessage(..), receive, sendBegin, sendError, sendReady, sendResult, sendSummary) + +import Json.Decode as Decode exposing (Decoder) +import Json.Encode as Encode + + +{-| The port names are prefixed to reduce the likelihood of the project +having a port with the same name, which is a compile error. +-} +port elmTestPort__send : Decode.Value -> Cmd msg + + +port elmTestPort__receive : (Decode.Value -> msg) -> Sub msg + + +sendBegin : Int -> Decode.Value -> Maybe Decode.Value -> Cmd msg +sendBegin initialSeed debugLogs maybeReport = + let + extraFields = + case maybeReport of + Just report -> + -- Test reporter specific: + [ ( "message", report ) ] + + Nothing -> + [] + in + elmTestPort__send + (Encode.object + (( "type", Encode.string "BEGIN" ) + :: ( "initialSeed", Encode.int initialSeed ) + :: ( "debugLogs", debugLogs ) + :: extraFields + ) + ) + + +sendReady : List Int -> List Int -> Cmd msg +sendReady unitTests fuzzTests = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "READY" ) + , ( "unitTests", Encode.list Encode.int unitTests ) + , ( "fuzzTests", Encode.list Encode.int fuzzTests ) + ] + ) + + +sendResult : Int -> Bool -> String -> List String -> Maybe String -> Decode.Value -> Decode.Value -> Cmd msg +sendResult testId isFuzzTest jsDefinitionName labels expectationElmCode debugLogs report = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "RESULT" ) + , ( "testId", Encode.int testId ) + , ( "testType" + , Encode.string + (if isFuzzTest then + "fuzz" + + else + "unit" + ) + ) + , ( "jsDefinitionName", Encode.string jsDefinitionName ) + , ( "labels", Encode.list Encode.string labels ) + , ( "expectationElmCode", encodeMaybe Encode.string expectationElmCode ) + , ( "debugLogs", debugLogs ) + + -- Test reporter specific: + , ( "message", report ) + ] + ) + + +sendSummary : Int -> Decode.Value -> Cmd msg +sendSummary exitCode summary = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "SUMMARY" ) + , ( "exitCode", Encode.int exitCode ) + + -- Test reporter specific: + , ( "message", summary ) + ] + ) + + +sendError : String -> Cmd msg +sendError message = + elmTestPort__send + (Encode.object + [ ( "type", Encode.string "ERROR" ) + , ( "message", Encode.string message ) + ] + ) + + +encodeMaybe : (a -> Encode.Value) -> Maybe a -> Encode.Value +encodeMaybe encoder maybe = + case maybe of + Just a -> + encoder a + + Nothing -> + Encode.null + + +type JsMessage + = RunUnitTest Int + | RunFuzzTest Int + | Summary Float Int (List ( List String, String )) + + +decoder : Decoder JsMessage +decoder = + Decode.field "type" Decode.string + |> Decode.andThen decodeMessageFromType + + +decodeMessageFromType : String -> Decoder JsMessage +decodeMessageFromType messageType = + case messageType of + "UNIT" -> + Decode.map RunUnitTest + (Decode.field "testId" Decode.int) + + "FUZZ" -> + Decode.map RunFuzzTest + (Decode.field "testId" Decode.int) + + "SUMMARY" -> + Decode.map3 Summary + (Decode.field "duration" Decode.float) + (Decode.field "failures" Decode.int) + (Decode.field "todos" (Decode.list todoDecoder)) + + _ -> + Decode.fail ("Unrecognized message type: " ++ messageType) + + +todoDecoder : Decoder ( List String, String ) +todoDecoder = + Decode.map2 (\a b -> ( a, b )) + (Decode.field "labels" (Decode.list Decode.string)) + (Decode.field "todo" Decode.string) + + +receive : (Result Decode.Error JsMessage -> msg) -> Sub msg +receive toMsg = + elmTestPort__receive (Decode.decodeValue decoder >> toMsg) diff --git a/lib/Generate.js b/lib/Generate.js index 143043f4..ff1046dd 100644 --- a/lib/Generate.js +++ b/lib/Generate.js @@ -2,6 +2,7 @@ const { supportsColor } = require('./chalk'); const fs = require('fs'); const path = require('path'); const ElmJson = require('./ElmJson'); +const Hash = require('./Hash'); const Solve = require('./Solve'); const before = fs.readFileSync( @@ -15,21 +16,41 @@ const after = fs.readFileSync( ); /** + * @param { Array<{ + moduleName: string, + possiblyTests: Array, + }> } testModules * @param { string } pipeFilename * @param { string } dest - * @returns { void } + * @param { boolean } unbufferedLogs + * @returns { Record } */ -function prepareCompiledJsFile(pipeFilename, dest) { +function prepareCompiledJsFile( + testModules, + pipeFilename, + dest, + unbufferedLogs +) { const content = fs.readFileSync(dest, 'utf8'); + + const names = testModules.flatMap((mod) => + mod.possiblyTests.map((test) => + toCompiledJavaScriptName(mod.moduleName, test) + ) + ); + + const hashes = Hash.calculateHashes(unbufferedLogs, names, content); + const finalContent = ` ${before} var Elm = (function() { -${patch(content)} +${patch(hashes, unbufferedLogs, content)} return this.Elm; }).call({}); var pipeFilename = ${JSON.stringify(pipeFilename)}; ${after} `.trim(); + fs.writeFileSync(dest, finalContent); // Needed when the user has `"type": "module"` in their package.json. @@ -38,19 +59,52 @@ ${after} path.join(path.dirname(dest), 'package.json'), JSON.stringify({ type: 'commonjs' }) ); + + return hashes; } -// For older versions of elm-explorations/test we need to list every single -// variant of the `Test` type. To avoid having to update this regex if a new -// variant is added, newer versions of elm-explorations/test have prefixed all -// variants with `ElmTestVariant__` so we can match just on that. +// To avoid having to update this regex if a new variant is added, +// elm-explorations/test have prefixed all variants with `ElmTestVariant__`. // `\$?` is for the Lamdera compiler, where definitions sometimes end with a `$`. // See https://github.com/lamdera/compiler/pull/41#issuecomment-2725158568 const testVariantDefinition = - /^var\s+\$elm_explorations\$test\$Test\$Internal\$(?:ElmTestVariant__\w+|UnitTest|FuzzTest|Labeled|Skipped|Only|Batch)\$?\s*=\s*(?:\w+\(\s*)?function\s*\([\w, ]*\)\s*\{\s*return *\{/gm; + /^var \$elm_explorations\$test\$Test\$Internal\$(?:ElmTestVariant__\w+)\$? = (?:F\d\(\s*)?function \([\w, ]*\) \{\s*return \{/gm; + +/** + * @param { string } name + * @returns { RegExp } + */ +function placeholderDefinition(name) { + return RegExp( + String.raw`^(var \$author\$project\$Test\$Runner\$Node\$${name}) = \$author\$project\$Test\$Runner\$Node\$placeholderReplaceMe___\('[^']+'\)`, + 'm' + ); +} + +const checkDefinition = placeholderDefinition('check'); +const getAndClearDebugLogsDefinition = placeholderDefinition( + 'getAndClearDebugLogs' +); +const getHashDefinition = placeholderDefinition('getHash'); +const runWithDurationDefinition = placeholderDefinition('runWithDuration'); -const checkDefinition = - /^(var\s+\$author\$project\$Test\$Runner\$Node\$check)\s*=\s*\$author\$project\$Test\$Runner\$Node\$checkHelperReplaceMe___;?$/m; +const debugLogDefinition = + /^var _Debug_log = F2\(function\(tag, value\)\s*\{[^}]+\}\)/m; + +const encodeEmptyDebugLogsCall = + /^(\s*)\$author\$project\$Test\$Generated\$PreviousRun\$encodeDebugLogs\(_List_Nil\)/gm; + +const encodeDebugLogsCall = + /^(\s*)\$author\$project\$Test\$Generated\$PreviousRun\$encodeDebugLogs\(\s*_List_fromArray\(/gm; + +/** + * @param { string } moduleName + * @param { string } valueName + * @returns { string } + */ +function toCompiledJavaScriptName(moduleName, valueName) { + return `$author$project$${moduleName.replace(/\./g, '$')}$${valueName}`; +} /** * Patch the JavaScript output from Elm: @@ -60,10 +114,12 @@ const checkDefinition = * - Silence `console.warn('Compiled in DEV mode. ...')`. The call is near the top of the file, * and the first usage of `console.warn`. * + * @param { Record } hashes + * @param { boolean } unbufferedLogs * @param { string } content * @returns { string } */ -function patch(content) { +function patch(hashes, unbufferedLogs, content) { return ( 'var __elmTestSymbol = Symbol("elmTestSymbol");\n' + content @@ -74,6 +130,58 @@ function patch(content) { ) // Simply remove the first occurrence of `console.warn`. This leaves the message string in parentheses behind, but that’s fine. .replace('console.warn', '') + .replace( + debugLogDefinition, + unbufferedLogs + ? ` +var _Debug_logs = []; +var _Debug_logPaused = false; +var _Debug_log = F2(function(tag, value) +{ + if (_Debug_logs.length === 0) { + _Debug_logs.push(''); + } + console.error(tag + ': ' + _Debug_toString(value)); + return value; +}); + `.trim() + : ` +var _Debug_logs = []; +var _Debug_logPaused = false; +var _Debug_logPausedMessage = 'For passing fuzz tests, Debug.log is not shown, since showing logs from lots of runs is pretty confusing. Tip: Use Debug.todo to fail a test from anywhere.'; +var _Debug_log = F2(function(tag, value) +{ + if (_Debug_logPaused) { + if (_Debug_logs.length === 0) { + _Debug_logs.push(_Debug_logPausedMessage); + } + } else { + _Debug_logs.push(tag + ': ' + _Debug_toString(value)); + } + return value; +}); + `.trim() + ) + .replace( + getAndClearDebugLogsDefinition, + '$1 = paused => { var logs = _Json_wrap(_Debug_logs); _Debug_logs = []; _Debug_logPaused = paused; return logs; }' + ) + .replace( + getHashDefinition, + `var __elmTestHashes = ${JSON.stringify( + hashes, + null, + 2 + )};\n$1 = name => __elmTestHashes[name]` + ) + .replace( + runWithDurationDefinition, + '$1 = thunk => { var t = performance.now(); return _Utils_Tuple2(thunk(null), performance.now() - t); }' + ) + // This optimizes a tiny bit: Instead of having a JS array, turning it into an Elm list, + // and then encoding it back to a JS array again, we just wrap the original array. + .replace(encodeEmptyDebugLogsCall, '$1_Json_wrap([])') + .replace(encodeDebugLogsCall, '$1(_Json_wrap(') ); } @@ -161,15 +269,20 @@ function generateElmJson( } } +const mainModuleName = ['Test', 'Generated', 'Main']; +const previousRunModuleName = ['Test', 'Generated', 'PreviousRun']; + /** - * @param { string } generatedCodeDir - * @returns { { + * @typedef { { moduleName: string, path: string, - } } + } } Module + * + * @param { string } generatedCodeDir + * @param { Array } moduleName + * @returns { Module } */ -function getMainModule(generatedCodeDir) { - const moduleName = ['Test', 'Generated', 'Main']; +function getModule(generatedCodeDir, moduleName) { return { moduleName: moduleName.join('.'), path: @@ -182,7 +295,7 @@ function getMainModule(generatedCodeDir) { /** * @param { number } fuzz - * @param { number } seed + * @param { number | null } seed * @param { import('./Report').Report } report * @param { Array } testFileGlobs * @param { Array } testFilePaths @@ -190,8 +303,7 @@ function getMainModule(generatedCodeDir) { moduleName: string, possiblyTests: Array, }> } testModules - * @param { { moduleName: string, path: string } } mainModule - * @param { number } processes + * @param { Module } mainModule * @returns { void } */ function generateMainModule( @@ -201,12 +313,11 @@ function generateMainModule( testFileGlobs, testFilePaths, testModules, - mainModule, - processes + mainModule ) { const testFileBody = makeTestFileBody( testModules, - makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths, processes) + makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) ); const testFileContents = `module ${mainModule.moduleName} exposing (main)\n\n${testFileBody}`; @@ -232,10 +343,12 @@ function makeTestFileBody(testModules, optsCode) { return ` ${imports.join('\n')} +import Dict import Test.Reporter.Reporter exposing (Report(..)) import Console.Text exposing (UseColor(..)) import Test.Runner.Node import Test +import ${previousRunModuleName.join('.')} main : Test.Runner.Node.TestProgram main = @@ -253,9 +366,10 @@ main = * @returns { string } */ function makeModuleTuple(mod) { - const list = mod.possiblyTests.map( - (test) => `Test.Runner.Node.check ${mod.moduleName}.${test}` - ); + const list = mod.possiblyTests.map((test) => { + const name = toCompiledJavaScriptName(mod.moduleName, test); + return `Test.Runner.Node.checkTagged ${mod.moduleName}.${test} "${name}"`; + }); return ` ( "${mod.moduleName}" @@ -298,26 +412,19 @@ function indentAllButFirstLine(indent, string) { /** * @param { number } fuzz - * @param { number } seed + * @param { number | null } seed * @param { import('./Report').Report } report * @param { Array } testFileGlobs * @param { Array } testFilePaths - * @param { number } processes * @returns { string } */ -function makeOptsCode( - fuzz, - seed, - report, - testFileGlobs, - testFilePaths, - processes -) { +function makeOptsCode(fuzz, seed, report, testFileGlobs, testFilePaths) { return ` { runs = ${fuzz} , report = ${generateElmReportVariant(report)} -, seed = ${seed} -, processes = ${processes} +, seed = ${seed === null ? makeRandomSeed() : seed} +, seedIsUserSupplied = ${seed === null ? 'False' : 'True'} +, previousRun = ${previousRunModuleName.join('.')}.previousRun , globs = ${indentAllButFirstLine(' ', makeList(testFileGlobs.map(makeElmString)))} , paths = @@ -326,6 +433,22 @@ function makeOptsCode( `.trim(); } +/** + * This will be passed to `Random.initialSeed`, which calls: + * `Bitwise.shiftRightZfBy 0 (incr + seed)` where `seed` is + * our number and `incr` is a constant. `Bitwise.shiftRightZfBy 0` + * is basically the same as as `modBy 0x100000000`. `incr` just shifts + * the numbers, it doesn’t affect how many different seeds there can be. + * In other words, there is no reason to pass a number higher than + * or equal to 0x100000000: After that we are just repeating seeds + * and have to be careful to not make some seeds more likely than others. + * + * @returns { number } + */ +function makeRandomSeed() { + return Math.floor(Math.random() * 0x100000000); +} + /** * @param { import('./Report').Report } report * @returns { string } @@ -356,9 +479,121 @@ function makeElmString(string) { .replace(/\r/g, '\\r')}"`; } +/** + * @param { Module } previousRunModule + * @returns { void } + */ +function ensurePreviousRunModule(previousRunModule) { + if (fs.existsSync(previousRunModule.path)) { + return; + } + generatePreviousRunModule(previousRunModule, { + fuzzRuns: -1, + initialSeed: -1, + cachedTests: {}, + }); +} + +/** + * @typedef { { + fuzzRuns: number, + initialSeed: number, + cachedTests: Record + } } PreviousRun + * + * @typedef { { + hash: string, + isActuallyTest: boolean, + unitTests: Array<{ labels: Array, expectation: string, debugLogs: Array }>, + fuzzTests: Array<{ labels: Array, expectation: string, debugLogs: Array }>, + } } CachedTests + * + * @param { Module } previousRunModule + * @param { PreviousRun } previousRun + * @returns { void } + */ +function generatePreviousRunModule(previousRunModule, previousRun) { + /** + * @param { { labels: Array, expectation: string, debugLogs: Array } } data + * @returns + */ + const toTuple = ({ labels, expectation, debugLogs }) => + ` +( ${indentAllButFirstLine(' ', makeList(labels.map(makeElmString)))} +, ( ${expectation} + , encodeDebugLogs + ${indentAllButFirstLine(' ', makeList(debugLogs.map(makeElmString)))} + ) +) + `.trim(); + + const cachedTestsList = makeList( + Object.entries(previousRun.cachedTests) + .filter(([, { isActuallyTest }]) => isActuallyTest) + .map(([jsIdentifierName, { hash, unitTests, fuzzTests }]) => + ` +( ${makeElmString(jsIdentifierName)} +, { hash = ${makeElmString(hash)} + , unitTests = + Dict.fromList + ${indentAllButFirstLine( + ' ', + makeList(unitTests.map(toTuple)) + )} + , fuzzTests = + Dict.fromList + ${indentAllButFirstLine( + ' ', + makeList(fuzzTests.map(toTuple)) + )} + } +) + `.trim() + ) + ); + + const fileContents = ` +module ${previousRunModule.moduleName} exposing (previousRun) + +import Dict +import Json.Encode +import Test.Distribution exposing (DistributionReport(..)) +import Test.Runner.Failure exposing (Reason(..), InvalidReason(..)) +import Test.Runner.Node exposing (CachedFuzzTestExpectation(..), CachedUnitTestExpectation(..)) + + +encodeDebugLogs : List String -> Json.Encode.Value +encodeDebugLogs = + Json.Encode.list Json.Encode.string + + +previousRun : Test.Runner.Node.PreviousRun +previousRun = + { fuzzRuns = ${previousRun.fuzzRuns} + , initialSeed = ${previousRun.initialSeed} + , cachedTests = + Dict.fromList + ${indentAllButFirstLine(' ', cachedTestsList)} + } + `.trim(); + + fs.mkdirSync(path.dirname(previousRunModule.path), { recursive: true }); + + // Write to a temporary file and then rename it atomically to the actual path. + // This avoids ending up with an empty file is elm-test is killed right between + // the file is truncated and written to. The tests sometimes failed due to this. + const tempPath = previousRunModule.path + '.tmp'; + fs.writeFileSync(tempPath, fileContents); + fs.renameSync(tempPath, previousRunModule.path); +} + module.exports = { + ensurePreviousRunModule: ensurePreviousRunModule, generateElmJson: generateElmJson, generateMainModule: generateMainModule, - getMainModule: getMainModule, + generatePreviousRunModule: generatePreviousRunModule, + getModule: getModule, + mainModuleName: mainModuleName, prepareCompiledJsFile: prepareCompiledJsFile, + previousRunModuleName: previousRunModuleName, }; diff --git a/lib/Hash.js b/lib/Hash.js new file mode 100644 index 00000000..6857368d --- /dev/null +++ b/lib/Hash.js @@ -0,0 +1,226 @@ +const crypto = require('crypto'); +const Tarjan = require('./Tarjan'); + +/** + * Pass in an array of definition names, such as `["$author$project$MyTest$suite"]`, + * which may be tests. For each definition name, find the corresponding + * JavaScript definition in the compiled Elm JavaScript code, and return a + * hash of the code of that definition. If the definition refers to other + * definitions (it calls other functions), the hash is based on both the hash + * of the code of the definition, and of the hashes of all referenced definitions. + * + * This way we can tell if the code that will be running via an exposed `Test` + * value has changed or not, and thus if we need to re-run it or not. + * + * @param { boolean } unbufferedLogs + * @param { Array } names + * @param { string } code + * @returns { Record } + */ +function calculateHashes(unbufferedLogs, names, code) { + const chunks = parseStep(code); + if (unbufferedLogs) { + chunks['_Debug_log'] += '/* unbuffered */'; + } + const graph = graphStep(chunks); + makeAcyclicStep(graph); + return hashStep(names, chunks, graph); +} + +/** + * The compiled Elm JavaScript is basically just a long sequence of definitions. + * Some are `function` statements, some are `var` assignments. + * + * Values that use themselves inside themselves in certain ways are defined + * with `function $some$module$cyclic$functionName`, and wrapped in `try {}` + * during development – that’s the only time a definition can be indented. + * + * We also need to support a `}` at the start of the line, because I’ve seen this + * code being generated (in https://github.com/lydell/codebase-ui/tree/02eac5056da3283687e0b61fa94a30ca6f71e3fb): + * + * function _Http_track(router, xhr, tracker) + * { + * // stuff + * }var $author$project$PreApp$AppMsg = function (a) { + * return {$: 'AppMsg', a: a}; + * }; + */ +const CHUNK_REGEX = /^(?=\}?(?:var|function|try))/m; + +/** + * Companion to `CHUNK_REGEX`. Extracts the name of the thing being defined a chunk. + * Remember that the chunk may start with `try` and be indented. + */ +const CHUNK_DEFINITION_NAME_REGEX = /(?:var|function) ([^ (]+)/; + +/** + * Matches string literals, multiline comments, singleline comments and some identifiers - + * which may be references to other chunks. Such references must start with either a + * dollar sign or an underscore. We only care about out the identifiers, but match the + * other literals too, so that we don’t get false positives for identifiers inside strings and comments. + * Parts copied from: https://github.com/lydell/js-tokens/blob/895fb4d6804a287aecfb0e1009851f925d07b079/index.coffee + * A more exact regex for identifiers is `/[$_][$_\u200C\u200D\p{ID_Continue}]+/gu`, + * but the one we’re using is about twice as fast. We match ASCII identifier chars, + * and then _anything_ non-ASCII, because the only non-ASCII characters outside strings + * and comments are going to be identifiers. + */ +const REFERENCES_REGEX = + /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?|\/\*(?:[^*]+|\*(?!\/))*(\*\/)?|\/\/.*|[$_][$\w\u0080-\uffff]+/g; + +/** + * Splits `code` into chunks as defined by `CHUNK_REGEX`. + * Returns the chunks that contain a definition (variable or function), + * keyed by the definition name. + * + * @param { string } code + * @returns { Record } + */ +function parseStep(code) { + /** @type { Record } */ + const chunks = {}; + for (const chunk of code.split(CHUNK_REGEX)) { + const match = CHUNK_DEFINITION_NAME_REGEX.exec(chunk); + // Not all chunks contain a definition. + if (match !== null) { + const name = match[1]; + chunks[name] = chunk; + } + } + return chunks; +} + +/** + * Parses references in all `chunks`. + * + * @typedef { { + definitions: Set, + references: Set, + } } Node + * + * @param { Record } chunks + * @returns { Record } + */ +function graphStep(chunks) { + /** @type { Record } */ + const graph = {}; + + for (const name in chunks) { + const chunk = chunks[name]; + const tokens = chunk.match(REFERENCES_REGEX); + graph[name] = { + definitions: new Set([name]), + references: + // Not all chunks contains any tokens that we care about (such as the `F` helper). + tokens === null + ? new Set() + : new Set( + tokens.filter( + (token) => + // Skip string literals and comments and take only identifiers – see `REFERENCES_REGEX`. + (token.startsWith('$') || token.startsWith('_')) && + // Skip direct recursion. + token !== name && + // Only care about references to stuff defined in `chunks`. + token in chunks + ) + ), + }; + } + + return graph; +} + +/** + * Merges recursive chains into single items, so that the output is an acyclic graph. + * + * @param { Record } graph + * @returns { void } + */ +function makeAcyclicStep(graph) { + const scc = Tarjan.stronglyConnectedComponents({ + keys: () => Object.keys(graph), + get: (key) => graph[key].references, + }); + + for (const chain of scc) { + if (chain.size > 1) { + // A chain of indirect recursion was found! + // Replace all the involved functions with the same node, + // containing the definitions and references of all the involved functions. + // This is how we make the graph acyclic. + /** @type { Set } */ + const references = new Set(); + for (const chainName of chain) { + // Note: `chainName` comes from keys in the graph. + const chainNode = graph[chainName]; + for (const chainReference of chainNode.references) { + // Skip direct recursion. + if (!chain.has(chainReference)) { + references.add(chainReference); + } + } + graph[chainName] = { + definitions: chain, + references, + }; + } + } + } +} + +/** + * @param { Array } names + * @param { Record } chunks + * @param { Record } graph + * @returns { Record } + */ +function hashStep(names, chunks, graph) { + /** @type { Record } */ + const hashes = {}; + + /** + * @param { string } name + * @returns { string } + */ + const getOrCalculateHash = (name) => { + // Already processed. + const hash = hashes[name]; + if (hash !== undefined) { + return hash; + } + + const node = graph[name]; + if (node === undefined) { + throw new Error( + `Could not find ${name} in the graph of the compiled code!` + ); + } + + // When testing on a large project, all hashes led to about the same + // amount of time used by `getOrCalculateHash`. `sha256` is one of + // the ones being about 10 ms faster than the slowest ones. + const hashObject = crypto.createHash('sha256'); + for (const name of Array.from(node.definitions).sort()) { + // Note: Nodes in `graph` only refer to things that exist in `chunks`. + hashObject.update(chunks[name]); + } + for (const reference of Array.from(node.references).sort()) { + hashObject.update(getOrCalculateHash(reference)); + } + + const newHash = hashObject.digest('hex'); + hashes[name] = newHash; + return newHash; + }; + + /** @type { Record } */ + const result = {}; + for (const name of names) { + result[name] = getOrCalculateHash(name); + } + return result; +} + +module.exports = { + calculateHashes, +}; diff --git a/lib/RunTests.js b/lib/RunTests.js index fc257a4d..25a2cd40 100644 --- a/lib/RunTests.js +++ b/lib/RunTests.js @@ -141,8 +141,9 @@ function watcherEventMessage(queue) { * @typedef { { watch: boolean, clearConsole: boolean, + unbufferedLogs: boolean, report: import('./Report').Report, - seed: number, + seed: number | null, fuzz: number, dependencies: import('./DependencyProvider').PackageStrategy, offline: boolean, @@ -165,6 +166,7 @@ function runTests( { watch, clearConsole, + unbufferedLogs, report, seed, fuzz, @@ -249,7 +251,14 @@ function runTests( runsExecuted++; const pipeFilename = getPipeFilename(runsExecuted); const testModules = FindTests.findTests(testFilePaths, project); - const mainModule = Generate.getMainModule(project.generatedCodeDir); + const mainModule = Generate.getModule( + project.generatedCodeDir, + Generate.mainModuleName + ); + const previousRunModule = Generate.getModule( + project.generatedCodeDir, + Generate.previousRunModuleName + ); const dest = path.join(project.generatedCodeDir, 'elmTestOutput.js'); Generate.generateElmJson( @@ -271,9 +280,9 @@ function runTests( testFileGlobs, testFilePaths, testModules, - mainModule, - processes + mainModule ); + Generate.ensurePreviousRunModule(previousRunModule); await Compile.compile( project.generatedCodeDir, @@ -283,16 +292,25 @@ function runTests( report ); - Generate.prepareCompiledJsFile(pipeFilename, dest); + const hashes = Generate.prepareCompiledJsFile( + testModules, + pipeFilename, + dest, + unbufferedLogs + ); progressLogger.log('Starting tests'); progressLogger.newLine(); return await Supervisor.run( packageInfo.version, + hashes, + previousRunModule, pipeFilename, + fuzz, report, processes, + unbufferedLogs, dest, watch ); diff --git a/lib/Supervisor.js b/lib/Supervisor.js index ff59bb2b..a31649c9 100644 --- a/lib/Supervisor.js +++ b/lib/Supervisor.js @@ -3,32 +3,72 @@ const child_process = require('child_process'); const fs = require('fs'); const net = require('net'); const readline = require('readline'); +const Generate = require('./Generate'); const Report = require('./Report'); const XMLBuilder = require('./XMLBuilder'); /** * @param { string } elmTestVersion + * @param { Record } hashes + * @param { import('./Generate').Module } previousRunModule * @param { string } pipeFilename + * @param { number } fuzz * @param { import('./Report').Report } report * @param { number } processes + * @param { boolean } unbufferedLogs * @param { string } dest * @param { boolean } watch * @returns { Promise } */ -function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { +function run( + elmTestVersion, + hashes, + previousRunModule, + pipeFilename, + fuzz, + report, + processes, + unbufferedLogs, + dest, + watch +) { return new Promise(function (resolve) { - /** @type { number | null } */ - var nextResultToPrint = null; - var finishedWorkers = 0; + /** @type { Array | undefined } */ + var unitTests = undefined; + /** @type { Array | undefined } */ + var fuzzTests = undefined; + var nextUnitTest = 0; + var nextFuzzTest = 0; + var finishedUnitTests = 0; + var finishedFuzzTests = 0; var closedWorkers = 0; var results = new Map(); var failures = 0; /** @type { Array<{ labels: Array, todo: string }> } */ var todos = []; - var testsToRun = -1; var startingTime = Date.now(); /** @type { Array } */ var workers = []; + /** @type { import('net').Server | undefined } */ + var server = undefined; + /** @type { import('./Generate').PreviousRun } */ + var toBePreviousRun = { + fuzzRuns: fuzz, + // When running with a random seed, Node.elm might decide to use + // the same seed as the last run to reproduce a failure. + // This is replaced with the real value at BEGIN. + initialSeed: -1, + cachedTests: {}, + }; + for (var key in hashes) { + toBePreviousRun.cachedTests[key] = { + hash: hashes[key], + // We don’t know if exposed items are tests or not until runtime. + isActuallyTest: false, + unitTests: [], + fuzzTests: [], + }; + } /** * @param { number } exitCode @@ -55,9 +95,9 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { case 'complete': switch (result.status) { case 'pass': - // passed tests should be printed only if they contain distributionReport - if (result.distributionReport !== undefined) { - console.log(makeWindowsSafe(result.distributionReport)); + // passed tests should be printed only if they contain debug logs or a distributionReport + if (result.message !== undefined) { + console.log(makeWindowsSafe(result.message)); } break; case 'todo': @@ -88,24 +128,6 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { } } - function flushResults() { - // Only print any results if we're ready - that is, nextResultToPrint - // is no longer null. (BEGIN changes it from null to 0.) - if (nextResultToPrint !== null) { - var result = results.get(nextResultToPrint); - - while ( - // If there are no more results to print, then we're done. - nextResultToPrint < testsToRun && - // Otherwise, keep going until we have no result available to print. - typeof result !== 'undefined' - ) { - printResult(result); - nextResultToPrint++; - result = results.get(nextResultToPrint); - } - } - } function reportRuntimeException() { console.error( chalk.red( @@ -115,10 +137,11 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { } /** - * @param { any } response This `any` became explicit instead of implicit when migrating from Flow to TypeScript. + * @param { number } testId + * @param { any } result This `any` became explicit instead of implicit when migrating from Flow to TypeScript. * @returns { void } */ - function handleResults(response) { + function handleResult(testId, result) { // TODO print progress bar - e.g. "Running test 5 of 20" on a bar! // -- yikes, be careful though...test the scenario where test // authors put Debug.log in their tests - does that mess @@ -128,42 +151,39 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { // backtrack the line feed, so that if someone else does more // logging, it will overwrite our status update and that's ok? - Object.keys(response.results).forEach(function (index) { - var result = response.results[index]; - results.set(parseInt(index), result); + if (report === 'junit') { + results.set(testId, result); + } - switch (report) { - case 'console': - switch (result.status) { - case 'pass': - // It's a PASS; no need to take any action. - break; - case 'todo': - todos.push(result); - break; - case 'fail': - failures++; - break; - default: - throw new Error(`Unexpected result.status: ${result.status}`); - } - break; - case 'junit': - if (typeof result.failure !== 'undefined') { - failures++; - } - break; - case 'json': - if (result.status === 'fail') { + switch (report) { + case 'console': + switch (result.status) { + case 'pass': + // It's a PASS; no need to take any action. + break; + case 'todo': + todos.push(result); + break; + case 'fail': failures++; - } else if (result.status === 'todo') { - todos.push({ labels: result.labels, todo: result.failures[0] }); - } - break; - } - }); - - flushResults(); + break; + default: + throw new Error(`Unexpected result.status: ${result.status}`); + } + break; + case 'junit': + if (typeof result.failure !== 'undefined') { + failures++; + } + break; + case 'json': + if (result.status === 'fail') { + failures++; + } else if (result.status === 'todo') { + todos.push({ labels: result.labels, todo: result.failures[0] }); + } + break; + } } /** @@ -171,6 +191,17 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { * @returns { void } */ function initWorker(socket) { + if (fuzzTests === undefined) { + throw new Error( + `fuzzTests is undefined, even though we have started workers for fuzz tests!` + ); + } + + // Other workers might have exhausted all fuzz tests before this one even got a chance to start. + if (nextFuzzTest >= fuzzTests.length) { + return; + } + socket.setEncoding('utf8'); socket.setNoDelay(true); @@ -182,39 +213,80 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { crlfDelay: Infinity, }); + /** @type { SendToWorker } */ + const send = (message) => { + socket.write(JSON.stringify(message)); + }; + stream.on('line', function (data) { - handleResponse(JSON.parse(data), (message) => { - socket.write(JSON.stringify(message)); - }); + handleResponse(JSON.parse(data), send); + }); + + send({ + type: 'FUZZ', + testId: fuzzTests[nextFuzzTest++], }); } /** - * @param { any } response This `any` became explicit instead of implicit when extracting this function. - * @param { (message: any) => void } send + * @typedef { + | { + type: 'BEGIN', + initialSeed: number, + debugLogs: Array, + message?: any, + } + | { + type: 'READY', + unitTests: Array, + fuzzTests: Array, + } + | { + type: 'RESULT', + testId: number, + testType: 'unit' | 'fuzz', + jsDefinitionName: string, + labels: Array, + expectationElmCode: string | null, + debugLogs: Array, + message: any, + } + | { + type: 'SUMMARY', + exitCode: number, + message: any, + } + | { + type: 'ERROR', + message: string, + } + } FromWorkerMessage - Needs to be in sync with Ports.elm. + * + * @typedef { (message: ToWorkerMessage) => void } SendToWorker + * @typedef { + | { + type: 'UNIT', + testId: number, + } + | { + type: 'FUZZ', + testId: number, + } + | { + type: 'SUMMARY', + duration: number, + failures: number, + todos: Array<{ labels: Array, todo: string }>, + } + } ToWorkerMessage - Needs to be in sync with Ports.elm. + * + * @param { FromWorkerMessage } response + * @param { SendToWorker } send * @returns { void } */ function handleResponse(response, send) { switch (response.type) { - case 'FINISHED': - handleResults(response); - - // This worker found no tests remaining to run; it's finished! - finishedWorkers++; - - // If all the workers have finished (or we run single-threaded), print the summary. - if (finishedWorkers === workers.length || processes === 1) { - send({ - type: 'SUMMARY', - duration: Date.now() - startingTime, - failures: failures, - todos: todos, - }); - } - break; case 'SUMMARY': - flushResults(); - if (response.exitCode === 1) { // The tests could not even run. At the time of this writing, the // only case is “No exposed values of type Test found”. That @@ -232,6 +304,11 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { xml.testsuite.testcase = xml.testsuite.testcase.concat(values); console.log(XMLBuilder.toString(xml)); } + + Generate.generatePreviousRunModule( + previousRunModule, + toBePreviousRun + ); } // Close all the workers. @@ -240,8 +317,10 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { }); end(response.exitCode); break; + case 'BEGIN': - testsToRun = response.testCount; + // Store the seed actually chosen to be used in the end. + toBePreviousRun.initialSeed = response.initialSeed; if (!Report.isMachineReadable(report)) { var headline = 'elm-test ' + elmTestVersion; @@ -251,52 +330,185 @@ function run(elmTestVersion, pipeFilename, report, processes, dest, watch) { } printResult(response.message); + if (response.debugLogs.length > 0) { + for (const debugLog of response.debugLogs) { + console.error(debugLog); + } + if (report === 'console') { + console.error('\n'); + } + } + break; - // Now we're ready to print results! - nextResultToPrint = 0; + case 'READY': + unitTests = response.unitTests; + fuzzTests = response.fuzzTests; - flushResults(); + if (unitTests.length === 0 && fuzzTests.length === 0) { + sendToMainProcess({ + type: 'SUMMARY', + duration: Date.now() - startingTime, + failures: failures, + todos: todos, + }); + } else { + // If running multi-threaded, run fuzz tests on threads. + // Save one core for the main thread. + if (fuzzTests.length > 0) { + if (processes > 1) { + startWorkers(Math.min(processes - 1, fuzzTests.length)); + } else if (unitTests.length === 0) { + sendToMainProcess({ + type: 'FUZZ', + testId: fuzzTests[nextFuzzTest++], + }); + } + } + // Run unit tests in the main thread. + if (unitTests.length > 0) { + sendToMainProcess({ + type: 'UNIT', + testId: unitTests[nextUnitTest++], + }); + } + } break; - case 'RESULTS': - handleResults(response); + + case 'RESULT': { + handleResult(response.testId, response.message); + printResult(response.message); + if (response.debugLogs.length > 0 && !unbufferedLogs) { + if (report !== 'console') { + console.error(response.labels.slice().reverse().join(' > ')); + } + for (const debugLog of response.debugLogs) { + console.error(debugLog); + } + if (report === 'console') { + console.error('\n'); + } + } + + const cachedTests = + toBePreviousRun.cachedTests[response.jsDefinitionName]; + cachedTests.isActuallyTest = true; + const usedUnbufferedLogs = + unbufferedLogs && response.debugLogs.length > 0; + switch (response.testType) { + case 'unit': + finishedUnitTests++; + if (response.expectationElmCode !== null && !usedUnbufferedLogs) { + cachedTests.unitTests.push({ + labels: response.labels, + expectation: response.expectationElmCode, + debugLogs: response.debugLogs, + }); + } + break; + + case 'fuzz': + finishedFuzzTests++; + if (response.expectationElmCode !== null && !usedUnbufferedLogs) { + cachedTests.fuzzTests.push({ + labels: response.labels, + expectation: response.expectationElmCode, + debugLogs: response.debugLogs, + }); + } + break; + } + + if (unitTests === undefined || fuzzTests === undefined) { + // Not READY yet. + break; + } + + if ( + finishedUnitTests >= unitTests.length && + finishedFuzzTests >= fuzzTests.length + ) { + sendToMainProcess({ + type: 'SUMMARY', + duration: Date.now() - startingTime, + failures: failures, + todos: todos, + }); + } else { + switch (response.testType) { + case 'unit': + if (nextUnitTest < unitTests.length) { + send({ + type: 'UNIT', + testId: unitTests[nextUnitTest++], + }); + } else if (processes === 1 && nextFuzzTest < fuzzTests.length) { + send({ + type: 'FUZZ', + testId: fuzzTests[nextFuzzTest++], + }); + } + break; + + case 'fuzz': + if (nextFuzzTest < fuzzTests.length) { + send({ + type: 'FUZZ', + testId: fuzzTests[nextFuzzTest++], + }); + } + break; + } + } break; + } + case 'ERROR': throw new Error(response.message); + default: - throw new Error('Unrecognized message from worker:' + response.type); + throw new Error( + 'Unrecognized message from worker: ' + + /** @type { { type: string } } */ (response).type + ); } } - // If just one process, run single-threaded. - if (processes === 1) { - var { run } = require(dest); - // Allow the generated file to be `require`d again (for watch mode). - delete require.cache[dest]; - var send = run( - 0, - /** @type { (response: any) => void } */ - (response) => { - handleResponse(response, send); - } - ); - } else { + /** @type { SendToWorker } */ + var sendToMainProcess = require(dest).run( + /* shouldSendBegin */ true, + /** @type { (response: FromWorkerMessage) => void } */ + (response) => { + handleResponse(response, sendToMainProcess); + } + ); + + // Allow the generated file to be `require`d again (for watch mode). + delete require.cache[dest]; + + /** + * @param { number } amount + * @returns { void } + */ + function startWorkers(amount) { var pendingException = false; // Using a named pipe to communicate is actually faster than // using `process.send` or `worker_threads`! See: // https://github.com/rtfeldman/node-test-runner/pull/674 - var server = net.createServer(initWorker); + server = net.createServer(initWorker); server.on('error', function (err) { console.error(err.stack); - server.close(); + if (server) { + server.close(); + } }); server.on('listening', function () { - workers = Array.from({ length: processes }, (_, index) => { - var worker = child_process.fork(dest, [index.toString()]); + workers = Array.from({ length: amount }, () => { + var worker = child_process.fork(dest); worker.on('close', function (code) { // code can be null. diff --git a/lib/Tarjan.js b/lib/Tarjan.js new file mode 100644 index 00000000..81c1cd2d --- /dev/null +++ b/lib/Tarjan.js @@ -0,0 +1,83 @@ +/** +Based on @rtsao/scc@1.1.0 +https://github.com/rtsao/scc/blob/317512b2b6615736ad9bd3f23e8cee739ff44cf6/index.js + +MIT License + +Copyright (c) 2019 Ryan Tsao + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +/** + * Find strongly connected components (SCC) of a directed graph using Tarjan's algorithm. + * + * Adapted from https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm#The_algorithm_in_pseudocode + * + * @typedef { { + keys: () => Array, + get: (key: string) => Set, + } } Graph + * + * @param { Graph } graph + * @returns { Array> } + */ +function stronglyConnectedComponents(graph) { + const indices = new Map(); + const lowLinks = new Map(); + const onStack = new Set(); + /** @type { Array } */ + const stack = []; + /** @type { Array> } */ + const scc = []; + let idx = 0; + + /** + * @param { string } v + * @returns { void } + */ + function strongConnect(v) { + indices.set(v, idx); + lowLinks.set(v, idx); + idx++; + stack.push(v); + onStack.add(v); + + const deps = graph.get(v); + for (const dep of deps) { + if (!indices.has(dep)) { + strongConnect(dep); + lowLinks.set(v, Math.min(lowLinks.get(v), lowLinks.get(dep))); + } else if (onStack.has(dep)) { + lowLinks.set(v, Math.min(lowLinks.get(v), indices.get(dep))); + } + } + + if (lowLinks.get(v) === indices.get(v)) { + const vertices = new Set(); + let w = null; + while (v !== w) { + w = stack.pop(); + onStack.delete(w); + vertices.add(w); + } + scc.push(vertices); + } + } + + for (const v of graph.keys()) { + if (!indices.has(v)) { + strongConnect(v); + } + } + + return scc; +} + +module.exports = { + stronglyConnectedComponents, +}; diff --git a/lib/elm-test.js b/lib/elm-test.js index 865f4dba..30d508d4 100644 --- a/lib/elm-test.js +++ b/lib/elm-test.js @@ -162,12 +162,17 @@ function main() { '--no-clear-console', "Don't clear the console when running with --watch" ) + .option( + '--unbuffered-logs', + 'Print debug logs immediately instead of at the end of each test', + false + ) // For example `--seed` and `--fuzz` only make sense for the “tests” command // and could be specified for that command only, but then they won’t show up // in `--help`. .addOption( new Option('--seed ', 'Run with a specific fuzzer seed') - .default(Math.floor(Math.random() * 407199254740991) + 1000, 'random') + .default(null, 'random') .argParser(parsePositiveInteger(0)) ) .option( diff --git a/templates/after.js b/templates/after.js index c9e57d5b..86f4a33c 100644 --- a/templates/after.js +++ b/templates/after.js @@ -1,5 +1,5 @@ -function run(index, receive) { - var app = Elm.Test.Generated.Main.init({ flags: index }); +function run(shouldSendBegin, receive) { + var app = Elm.Test.Generated.Main.init({ flags: shouldSendBegin }); // Without this, each run leaks memory in single-threaded mode: Elm = null; app.ports.elmTestPort__send.subscribe(receive); @@ -19,7 +19,7 @@ function main() { client.setEncoding('utf8'); client.setNoDelay(true); - var send = run(Number(process.argv[2]), function (msg) { + var send = run(false, function (msg) { // We split incoming messages on the socket on newlines. The gist is that node // is rather unpredictable in whether or not a single `write` will result in a // single `on('data')` callback. Sometimes it does, sometimes multiple writes