test: Collect the test files in one place

test/blockchaintest/blockchaintest.cpp

@@ -7,6 +7,7 @@
 #include <evmone/version.h>
 #include <gtest/gtest.h>
 #include <test/utils/blockchaintest.hpp>
+#include <test/utils/test_files.hpp>
 #include <test/utils/test_report.hpp>
 #include <iostream>
 
@@ -91,17 +92,8 @@ void register_test_files(const fs::path& root, evmc::VM& vm)
 {
     if (is_directory(root))
     {
-        std::vector<fs::path> test_files;
-        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
-            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
-                // "index.json" files are just lists of tests generated by other tools.
-                return entry.is_regular_file() && entry.path().extension() == ".json" &&
-                       entry.path().filename() != "index.json";
-            });
-        std::ranges::sort(test_files);
-
-        for (const auto& p : test_files)
-            BlockchainGTestFile::register_one(fs::relative(p, root).parent_path().string(), p, vm);
+        for (const auto& [path, suite_name] : evmone::test::collect_test_files(root))
+            BlockchainGTestFile::register_one(suite_name, path, vm);
     }
     else  // Treat as a file.
     {

test/integration/blockchaintest/CMakeLists.txt

@@ -17,10 +17,10 @@ set_tests_properties(
     PASS_REGULAR_EXPRESSION ".*2 tests from"
 )
 
-# Exercise block-level gas accounting (EIP-7778).
+# Exercise block-level gas accounting (EIP-7778), with tracing on so that flag is covered too.
 add_test(
     NAME ${PREFIX}/eip7778_block_gas
-    COMMAND evmone-blockchaintest ${TESTS1}/eip7778_block_gas.json
+    COMMAND evmone-blockchaintest ${TESTS1}/eip7778_block_gas.json --trace
 )
 
 # A block invalid by its signature must not be judged by another check.
@@ -33,5 +33,27 @@ add_test(
     COMMAND evmone-blockchaintest ${TESTS1}/unrecovered_sender_deposit_log.json
 )
 
+# A block with expectException but no rlp_decoded is a shape the loader does not support. As a
+# single file it is reported and nothing is registered.
+add_test(
+    NAME ${PREFIX}/unsupported_rlp
+    COMMAND evmone-blockchaintest ${TESTS1}/unsupported_rlp.json
+)
+set_tests_properties(
+    ${PREFIX}/unsupported_rlp PROPERTIES
+    PASS_REGULAR_EXPRESSION "tests with invalidly rlp-encoded blocks are not supported"
+)
+
+# The directory form registers one test per file, which none of the tests above reach, and skips
+# the unsupported one rather than failing on it.
+add_test(
+    NAME ${PREFIX}/directory
+    COMMAND evmone-blockchaintest ${TESTS1}
+)
+set_tests_properties(
+    ${PREFIX}/directory PROPERTIES
+    PASS_REGULAR_EXPRESSION "SKIPPED \\] \\.unsupported_rlp"
+)
+
 get_directory_property(ALL_TESTS TESTS)
 set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw)

test/integration/blockchaintest/unsupported_rlp.json

@@ -0,0 +1,47 @@
+{
+  "unsupported_rlp": {
+    "_info": {
+      "comment": "The runner takes the sender from the fixture, so it cannot see the broken signature; the transaction executes and the wrong blobGasUsed is the first header mismatch."
+    },
+    "network": "Cancun",
+    "genesisBlockHeader": {
+      "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+      "coinbase": "0x0000000000000000000000000000000000000000",
+      "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000000",
+      "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+      "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+      "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "number": "0x00",
+      "gasLimit": "0x01000000",
+      "gasUsed": "0x00",
+      "timestamp": "0x00",
+      "extraData": "0x00",
+      "baseFeePerGas": "0x10",
+      "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+      "blobGasUsed": "0x00",
+      "excessBlobGas": "0x00",
+      "hash": "0x0000000000000000000000000000000000000000000000000000000000000001"
+    },
+    "pre": {
+      "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+        "code": "0x",
+        "nonce": "0x00",
+        "balance": "0x02540be400"
+      }
+    },
+    "blocks": [
+      {
+        "expectException": "TransactionException.INVALID_SIGNATURE_VRS",
+        "rlp": "0x"
+      }
+    ],
+    "lastblockhash": "0x0000000000000000000000000000000000000000000000000000000000000001",
+    "postState": {
+      "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+        "code": "0x",
+        "nonce": "0x00",
+        "balance": "0x02540be400"
+      }
+    }
+  }
+}

test/integration/statetest/CMakeLists.txt

@@ -19,6 +19,8 @@ set_tests_properties(
     PASS_REGULAR_EXPRESSION "path is required"
 )
 
+# SuiteA also holds an index.json and a notes.txt, neither of which is a test. They sort before
+# test1.json, so collecting either would show up inside the SuiteA block below.
 add_test(
     NAME ${PREFIX}/tests1_list
     COMMAND evmone-statetest ${TESTS1} --gtest_list_tests
@@ -134,5 +136,18 @@ set_tests_properties(
     FAIL_REGULAR_EXPRESSION "failing_test_case"
 )
 
+# Over a directory the filter is applied per case inside the file's test, not at registration as
+# it is above. The summary line is what proves a case ran: forbidding the other name alone would
+# hold just as well if the filter dropped every case.
+add_test(
+    NAME ${PREFIX}/filter_directory
+    COMMAND evmone-statetest ${TESTS_FILTER} -k passing_test_case --trace-summary
+)
+set_tests_properties(
+    ${PREFIX}/filter_directory PROPERTIES
+    PASS_REGULAR_EXPRESSION "\"pass\":true"
+    FAIL_REGULAR_EXPRESSION "failing_test_case"
+)
+
 get_directory_property(ALL_TESTS TESTS)
 set_tests_properties(${ALL_TESTS} PROPERTIES ENVIRONMENT LLVM_PROFILE_FILE=${CMAKE_BINARY_DIR}/integration-%p.profraw)

test/integration/statetest/tests1/SuiteA/index.json

@@ -0,0 +1,4 @@
+{
+  "comment": "A listing generated by other tools, not a test. It must not be collected.",
+  "tests": ["test1.json", "test2_multi.json"]
+}

test/integration/statetest/tests1/SuiteA/notes.txt

@@ -0,0 +1 @@
+Not a test either: only .json files are collected.

test/statetest/statetest.cpp

@@ -7,6 +7,7 @@
 #include <evmone/version.h>
 #include <gtest/gtest.h>
 #include <test/utils/statetest.hpp>
+#include <test/utils/test_files.hpp>
 #include <test/utils/test_report.hpp>
 #include <iostream>
 
@@ -96,18 +97,8 @@ void register_test_files(
 {
     if (is_directory(root))
     {
-        std::vector<fs::path> test_files;
-        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
-            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
-                // "index.json" files are just lists of tests generated by other tools.
-                return entry.is_regular_file() && entry.path().extension() == ".json" &&
-                       entry.path().filename() != "index.json";
-            });
-        std::ranges::sort(test_files);
-
-        for (const auto& p : test_files)
-            StateTestFile::register_one(
-                fs::relative(p, root).parent_path().string(), p, filter, vm, trace);
+        for (const auto& [path, suite_name] : evmone::test::collect_test_files(root))
+            StateTestFile::register_one(suite_name, path, filter, vm, trace);
     }
     else  // Treat as a file.
     {

test/utils/CMakeLists.txt

@@ -40,6 +40,8 @@ target_sources(
     statetest_runner.cpp
     t8n.hpp
     t8n.cpp
+    test_files.hpp
+    test_files.cpp
     test_report.hpp
     test_report.cpp
     test_state.hpp

test/utils/test_files.cpp

@@ -0,0 +1,34 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2026 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "test_files.hpp"
+#include <algorithm>
+#include <ranges>
+
+namespace evmone::test
+{
+namespace fs = std::filesystem;
+
+std::vector<TestFile> collect_test_files(const fs::path& root)
+{
+    static constexpr auto is_test_file = [](const fs::directory_entry& entry) {
+        return entry.is_regular_file() && entry.path().extension() == ".json" &&
+               entry.path().filename() != "index.json";
+    };
+    const auto as_test_file = [&root](const fs::directory_entry& entry) {
+        return TestFile{entry.path(), fs::relative(entry.path(), root).parent_path().string()};
+    };
+    // TODO(gcc-12): Pipe the temporary in directly. Adapting one needs owning_view, which C++20
+    //   has but gcc-11's libstdc++ does not implement.
+    const fs::recursive_directory_iterator entries{root};
+
+    // TODO(C++23): std::ranges::to<std::vector>() replaces the vector and the copy.
+    std::vector<TestFile> files;
+    std::ranges::copy(
+        entries | std::views::filter(is_test_file) | std::views::transform(as_test_file),
+        std::back_inserter(files));
+    std::ranges::sort(files);
+    return files;
+}
+}  // namespace evmone::test

test/utils/test_files.hpp

@@ -0,0 +1,28 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2026 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace evmone::test
+{
+/// A test file found under a search root.
+struct TestFile
+{
+    std::filesystem::path path;
+
+    /// The file's directory relative to the root, which is the test suite it belongs to.
+    std::string suite_name;
+
+    /// Ordered by path, because that is the order a run visits them in. The suite name is derived
+    /// from the path, so it never gets to decide a comparison.
+    auto operator<=>(const TestFile&) const = default;
+};
+
+/// The JSON test files under @p root, sorted, so a run visits them in a stable order.
+/// "index.json" is skipped: those are lists of tests generated by other tools, not tests.
+[[nodiscard]] std::vector<TestFile> collect_test_files(const std::filesystem::path& root);
+}  // namespace evmone::test