diff --git a/circle.yml b/circle.yml index b0a3517208..58e49b6253 100644 --- a/circle.yml +++ b/circle.yml @@ -162,9 +162,10 @@ commands: fixtures_suffix: type: string default: "" - filter: + ignore: + description: "--ignore flags for the paths not to collect" type: string - default: "*" + default: "" steps: - download_execution_spec_tests: repo: <> @@ -176,14 +177,14 @@ commands: working_directory: ~/build command: > LLVM_PROFILE_FILE=state_tests.profraw - bin/evmone-statetest --gtest_filter='<>' ~/spec-tests/fixtures/state_tests + bin/evmone-statetest <> ~/spec-tests/fixtures/state_tests - run: name: "Execution spec tests (<>, blockchain_tests)" # Tests for in-development EVM revision currently passing. working_directory: ~/build command: > LLVM_PROFILE_FILE=blockchain_tests.profraw - bin/evmone-blockchaintest --gtest_filter='<>' ~/spec-tests/fixtures/blockchain_tests + bin/evmone-blockchaintest <> ~/spec-tests/fixtures/blockchain_tests configure: description: "Configure" @@ -467,7 +468,7 @@ jobs: - run_execution_spec_tests: release: tests-glamsterdam-devnet@v8.1.2 fixtures_suffix: _glamsterdam-devnet - filter: "-for_amsterdam/*:for_bpo2toamsterdamattime15k/*" + ignore: "--ignore for_amsterdam --ignore for_bpo2toamsterdamattime15k" - collect_coverage_clang: flags: eest-develop ignore_filename_regex: lib/evmone/(advanced|cpu_check|eof|lru_cache|tracing|vm)|test/(experimental|unittests) @@ -517,7 +518,9 @@ jobs: command: > LLVM_PROFILE_FILE=blockchain_tests_invalid.profraw bin/evmone-blockchaintest - --gtest_filter='-bc4895-withdrawals.shanghaiWithoutWithdrawalsRLP:bcInvalidHeaderTest.*:bcUncleHeaderValidity.gasLimitTooLowExactBound' + --ignore bc4895-withdrawals/shanghaiWithoutWithdrawalsRLP.json + --ignore bcInvalidHeaderTest + --ignore bcUncleHeaderValidity/gasLimitTooLowExactBound.json ~/tests/BlockchainTests/InvalidBlocks ~/tests/LegacyTests/Cancun/BlockchainTests/InvalidBlocks - collect_coverage_clang: diff --git a/test/blockchaintest/blockchaintest.cpp b/test/blockchaintest/blockchaintest.cpp index f816ec532b..f47eddd3b3 100644 --- a/test/blockchaintest/blockchaintest.cpp +++ b/test/blockchaintest/blockchaintest.cpp @@ -88,12 +88,21 @@ class BlockchainGTest : public testing::Test } }; -void register_test_files(const fs::path& root, evmc::VM& vm) +/// Registers every test under @p root, or prints its path if @p collect_only. +void register_test_files( + const fs::path& root, std::span ignored, bool collect_only, evmc::VM& vm) { if (is_directory(root)) { - for (const auto& [path, suite_name] : evmone::test::collect_test_files(root)) - BlockchainGTestFile::register_one(suite_name, path, vm); + auto files = evmone::test::collect_test_files(root); + evmone::test::ignore_test_files(files, ignored); + for (const auto& [path, suite_name] : files) + { + if (collect_only) + std::cout << path.string() << '\n'; + else + BlockchainGTestFile::register_one(suite_name, path, vm); + } } else // Treat as a file. { @@ -102,7 +111,12 @@ void register_test_files(const fs::path& root, evmc::VM& vm) { const auto tests = evmone::test::load_blockchain_tests(f); for (const auto& test : tests) - BlockchainGTest::register_one(test, root.string(), test.name, root, vm); + { + if (collect_only) + std::cout << root.string() << "::" << test.name << '\n'; + else + BlockchainGTest::register_one(test, root.string(), test.name, root, vm); + } } catch (const evmone::test::UnsupportedTestFeature& ex) { @@ -131,6 +145,18 @@ int main(int argc, char* argv[]) ->required() ->check(CLI::ExistingPath); + std::vector ignored; + app.add_option("--ignore", ignored, + "Path, relative to a test directory, not to collect tests from. May be given more " + "than once. Whole path components are matched, so --ignore bc4895 keeps " + "bc4895-withdrawals.") + // Without this the option is variadic and swallows the positional paths after it. + ->allow_extra_args(false); + + bool collect_only = false; + app.add_flag("--collect-only", collect_only, + "List the path of each collected test, one per line, and exit."); + bool trace_flag = false; app.add_flag("--trace", trace_flag, "Enable EVM tracing"); @@ -142,9 +168,9 @@ int main(int argc, char* argv[]) vm.set_option("trace", "1"); for (const auto& p : paths) - register_test_files(p, vm); + register_test_files(p, ignored, collect_only, vm); - return RUN_ALL_TESTS(); + return collect_only ? 0 : RUN_ALL_TESTS(); } catch (const std::exception& ex) { diff --git a/test/integration/blockchaintest/CMakeLists.txt b/test/integration/blockchaintest/CMakeLists.txt index ade6350d7a..fdafeddf6f 100644 --- a/test/integration/blockchaintest/CMakeLists.txt +++ b/test/integration/blockchaintest/CMakeLists.txt @@ -55,5 +55,27 @@ set_tests_properties( PASS_REGULAR_EXPRESSION "SKIPPED \\] \\.unsupported_rlp" ) +# --collect-only lists what would run instead of running it, and --ignore drops a path from that +# list. Over a directory a test is listed as its file. +add_test( + NAME ${PREFIX}/collect_only_directory + COMMAND evmone-blockchaintest ${TESTS1} --collect-only --ignore unsupported_rlp.json +) +set_tests_properties( + ${PREFIX}/collect_only_directory PROPERTIES + PASS_REGULAR_EXPRESSION "eip7778_block_gas\\.json" + FAIL_REGULAR_EXPRESSION "unsupported_rlp" +) + +# Given the file directly, each test case in it is listed on its own. +add_test( + NAME ${PREFIX}/collect_only_file + COMMAND evmone-blockchaintest ${TESTS1}/test.json --collect-only +) +set_tests_properties( + ${PREFIX}/collect_only_file PROPERTIES + PASS_REGULAR_EXPRESSION "test\\.json::[^\n]*-call\\]\n[^\n]*test\\.json::[^\n]*-callcode\\]" +) + get_directory_property(ALL_TESTS TESTS) set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw) diff --git a/test/integration/export/CMakeLists.txt b/test/integration/export/CMakeLists.txt index 6df940bef8..6659ef9ffd 100644 --- a/test/integration/export/CMakeLists.txt +++ b/test/integration/export/CMakeLists.txt @@ -31,7 +31,7 @@ set_tests_properties( add_test( NAME ${PREFIX}/execute_exported_state_tests # TODO: Broken exported tests are filtered out. - COMMAND evmone-statetest ${EXPORT_DIR}/state_tests --gtest_filter=-*block.* + COMMAND evmone-statetest ${EXPORT_DIR}/state_tests --ignore state_transition/block ) set_tests_properties( ${PREFIX}/execute_exported_state_tests PROPERTIES diff --git a/test/integration/statetest/CMakeLists.txt b/test/integration/statetest/CMakeLists.txt index 224fe2589f..0c023936e1 100644 --- a/test/integration/statetest/CMakeLists.txt +++ b/test/integration/statetest/CMakeLists.txt @@ -19,52 +19,37 @@ set_tests_properties( PASS_REGULAR_EXPRESSION "path is required" ) -# SuiteA also holds an index.json and a notes.txt, neither of which is a test. They sort before -# test1.json, so collecting either would show up inside the SuiteA block below. +# A test collected from a directory is listed as its file. SuiteA also holds an index.json and a +# notes.txt, neither of which is a test; both sort before test1.json, so collecting either would +# show up in the listing. add_test( NAME ${PREFIX}/tests1_list - COMMAND evmone-statetest ${TESTS1} --gtest_list_tests + COMMAND evmone-statetest ${TESTS1} --collect-only ) set_tests_properties( ${PREFIX}/tests1_list PROPERTIES - PASS_REGULAR_EXPRESSION [[ -B\. - T -SuiteA\. - test1 - test2_multi -]] + PASS_REGULAR_EXPRESSION "tests1[^\n]*T\\.json\n[^\n]*tests1[^\n]*test1\\.json\n[^\n]*tests1[^\n]*test2_multi\\.json" ) +# Given the file directly, each test case in it is listed on its own. add_test( NAME ${PREFIX}/single_file_list - COMMAND evmone-statetest ${TESTS1}/SuiteA/test1.json --gtest_list_tests + COMMAND evmone-statetest ${TESTS1}/SuiteA/test2_multi.json --collect-only ) set_tests_properties( ${PREFIX}/single_file_list PROPERTIES - PASS_REGULAR_EXPRESSION [[ -.*test/integration/statetest/tests1/SuiteA/test1\.json\. - test1 -]] + PASS_REGULAR_EXPRESSION "test2_multi\\.json::test_case_1\n[^\n]*test2_multi\\.json::test_case_2" ) +# Several roots are collected in the order given, not regrouped by suite as gtest listed them. +# T.json holds no test cases, so naming it directly contributes no line. add_test( NAME ${PREFIX}/multiple_args_list - COMMAND evmone-statetest ${TESTS1} ${TESTS2} ${TESTS1}/B/T.json ${TESTS1}/SuiteA --gtest_list_tests + COMMAND evmone-statetest ${TESTS1} ${TESTS2} ${TESTS1}/B/T.json ${TESTS1}/SuiteA --collect-only ) set_tests_properties( ${PREFIX}/multiple_args_list PROPERTIES - PASS_REGULAR_EXPRESSION [[ -B\. - T -SuiteA\. - test1 - test2_multi - test1 -\. - test1 - test2_multi -]] + PASS_REGULAR_EXPRESSION "tests1[^\n]*T\\.json\n[^\n]*tests1[^\n]*test1\\.json\n[^\n]*tests1[^\n]*test2_multi\\.json\n[^\n]*tests2[^\n]*test1\\.json\n[^\n]*tests1[^\n]*test1\\.json\n[^\n]*tests1[^\n]*test2_multi\\.json" ) add_test( @@ -149,5 +134,17 @@ set_tests_properties( FAIL_REGULAR_EXPRESSION "failing_test_case" ) +# --ignore drops a path from the collection. The flags come first, as they do in CI: an --ignore +# that swallowed a positional path would leave the roots after it uncollected. +add_test( + NAME ${PREFIX}/ignore + COMMAND evmone-statetest --ignore B ${TESTS1} ${TESTS2} --collect-only +) +set_tests_properties( + ${PREFIX}/ignore PROPERTIES + PASS_REGULAR_EXPRESSION "tests1[^\n]*test1\\.json\n[^\n]*tests1[^\n]*test2_multi\\.json\n[^\n]*tests2[^\n]*test1\\.json" + FAIL_REGULAR_EXPRESSION "T\\.json" +) + get_directory_property(ALL_TESTS TESTS) set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw) diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp index f1b0594a98..e17b4801a4 100644 --- a/test/statetest/statetest.cpp +++ b/test/statetest/statetest.cpp @@ -92,13 +92,21 @@ class StateTest : public testing::Test } }; -void register_test_files( - const fs::path& root, const std::optional& filter, evmc::VM& vm, bool trace) +/// Registers every test under @p root, or prints its path if @p collect_only. +void register_test_files(const fs::path& root, const std::optional& filter, + std::span ignored, bool collect_only, evmc::VM& vm, bool trace) { if (is_directory(root)) { - for (const auto& [path, suite_name] : evmone::test::collect_test_files(root)) - StateTestFile::register_one(suite_name, path, filter, vm, trace); + auto files = evmone::test::collect_test_files(root); + evmone::test::ignore_test_files(files, ignored); + for (const auto& [path, suite_name] : files) + { + if (collect_only) + std::cout << path.string() << '\n'; + else + StateTestFile::register_one(suite_name, path, filter, vm, trace); + } } else // Treat as a file. { @@ -108,7 +116,10 @@ void register_test_files( { if (filter.has_value() && test.name.find(*filter) == std::string::npos) continue; - StateTest::register_one(test, root.string(), test.name, root, vm, trace); + if (collect_only) + std::cout << root.string() << "::" << test.name << '\n'; + else + StateTest::register_one(test, root.string(), test.name, root, vm, trace); } } } @@ -117,17 +128,6 @@ void register_test_files( int main(int argc, char* argv[]) { - // The default test filter. To enable all tests use `--gtest_filter=*`. - testing::FLAGS_gtest_filter = - "-" - // Slow tests: - "stCreateTest.CreateOOGafterMaxCodesize:" // pass - "stQuadraticComplexityTest.Call50000_sha256:" // pass - "stTimeConsuming.static_Call50000_sha256:" // pass - "stTimeConsuming.CALLBlake2f_MaxRounds:" // pass - "VMTests/vmPerformance.*:" // pass - ; - try { testing::InitGoogleTest(&argc, argv); // Process GoogleTest flags. @@ -148,6 +148,18 @@ int main(int argc, char* argv[]) app.add_option("-k", filter, "Test name filter. Run only tests with names containing the specified string."); + std::vector ignored; + app.add_option("--ignore", ignored, + "Path, relative to a test directory, not to collect tests from. May be given more " + "than once. Whole path components are matched, so --ignore bc4895 keeps " + "bc4895-withdrawals.") + // Without this the option is variadic and swallows the positional paths after it. + ->allow_extra_args(false); + + bool collect_only = false; + app.add_flag("--collect-only", collect_only, + "List the path of each collected test, one per line, and exit."); + bool trace = false; bool trace_summary = false; const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing"); @@ -165,9 +177,9 @@ int main(int argc, char* argv[]) } for (const auto& p : paths) - register_test_files(p, filter, vm, trace || trace_summary); + register_test_files(p, filter, ignored, collect_only, vm, trace || trace_summary); - return RUN_ALL_TESTS(); + return collect_only ? 0 : RUN_ALL_TESTS(); } catch (const std::exception& ex) { diff --git a/test/unittests/CMakeLists.txt b/test/unittests/CMakeLists.txt index 34b4f7605a..c9db2581ea 100644 --- a/test/unittests/CMakeLists.txt +++ b/test/unittests/CMakeLists.txt @@ -85,6 +85,7 @@ target_sources( statetest_logs_hash_test.cpp statetest_runner_test.cpp statetest_withdrawals_test.cpp + test_files_test.cpp test_report_test.cpp tooling_run_test.cpp tooling_t8n_test.cpp diff --git a/test/unittests/test_files_test.cpp b/test/unittests/test_files_test.cpp new file mode 100644 index 0000000000..ae5bccb033 --- /dev/null +++ b/test/unittests/test_files_test.cpp @@ -0,0 +1,76 @@ +// evmone: Fast Ethereum Virtual Machine implementation +// Copyright 2026 The evmone Authors. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +using namespace evmone::test; +namespace fs = std::filesystem; + +namespace +{ +/// A collection as collect_test_files() would return it, with a sibling directory whose name +/// begins with another one's. +std::vector collected() +{ + return { + {"root/bc4895/a.json", "bc4895"}, + {"root/bc4895/nested/c.json", "bc4895/nested"}, + {"root/bc4895-withdrawals/b.json", "bc4895-withdrawals"}, + {"root/top.json", ""}, + }; +} + +std::vector names(const std::vector& files) +{ + std::vector result; + result.reserve(files.size()); + for (const auto& f : files) + result.push_back(f.path.filename().string()); + return result; +} +} // namespace + +TEST(test_files, ignore_nothing) +{ + const std::vector all{"a.json", "c.json", "b.json", "top.json"}; + + auto files = collected(); + ignore_test_files(files, {}); + EXPECT_EQ(names(files), all); + + // An empty path, which an unset variable expands to, must not drop everything. Neither must + // ".", which names the search root: pytest also collects it all for --ignore of the root. + const std::vector ignored{"", ".", "./"}; + ignore_test_files(files, ignored); + EXPECT_EQ(names(files), all); +} + +TEST(test_files, ignore_directory) +{ + // The sibling shares the prefix as text, but not as a path component. + auto files = collected(); + const std::vector ignored{"bc4895"}; + ignore_test_files(files, ignored); + EXPECT_EQ(names(files), (std::vector{"b.json", "top.json"})); +} + +TEST(test_files, ignore_directory_other_spellings) +{ + for (const auto& spelling : {"bc4895/", "./bc4895"}) + { + auto files = collected(); + const std::vector ignored{spelling}; + ignore_test_files(files, ignored); + EXPECT_EQ(names(files), (std::vector{"b.json", "top.json"})) << spelling; + } +} + +TEST(test_files, ignore_files) +{ + auto files = collected(); + const std::vector ignored{"bc4895/nested/c.json", "top.json"}; + ignore_test_files(files, ignored); + EXPECT_EQ(names(files), (std::vector{"a.json", "b.json"})); +} diff --git a/test/utils/test_files.cpp b/test/utils/test_files.cpp index ce1db39057..78921c0a4c 100644 --- a/test/utils/test_files.cpp +++ b/test/utils/test_files.cpp @@ -31,4 +31,27 @@ std::vector collect_test_files(const fs::path& root) std::ranges::sort(files); return files; } + +void ignore_test_files(std::vector& files, std::span ignored) +{ + // Whether the path begins with every component of the prefix. + static constexpr auto is_under = [](const fs::path& path, const fs::path& prefix) { + // "./B" has to name what "B" names, and a trailing separator, which tab completion adds, + // is an empty final component of its own. + auto p = prefix.lexically_normal(); + if (p.filename().empty()) + p = p.parent_path(); + // An empty prefix, which an unset variable expands to, names nothing rather than + // everything. + return !p.empty() && std::ranges::mismatch(p, path).in1 == p.end(); + }; + + std::erase_if(files, [ignored](const TestFile& file) { + // The suite name is the file's directory relative to the root, which is what the ignored + // paths are relative to as well. + const auto relative = fs::path{file.suite_name} / file.path.filename(); + return std::ranges::any_of( + ignored, [&relative](const fs::path& prefix) { return is_under(relative, prefix); }); + }); +} } // namespace evmone::test diff --git a/test/utils/test_files.hpp b/test/utils/test_files.hpp index 6f07e0c1ed..f116d6639e 100644 --- a/test/utils/test_files.hpp +++ b/test/utils/test_files.hpp @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include @@ -25,4 +26,10 @@ struct TestFile /// The JSON test files under @p root, sorted, so a run visits them in a stable order. /// "index.json" is skipped: those are lists of tests generated by other tools, not tests. [[nodiscard]] std::vector collect_test_files(const std::filesystem::path& root); + +/// Drops from @p files everything under one of the @p ignored paths, which are relative to the +/// search root. Whole path components are matched, so ignoring "bc4895" keeps +/// "bc4895-withdrawals". +void ignore_test_files( + std::vector& files, std::span ignored); } // namespace evmone::test