From f6514bb74d477620832a804d649736fdd880020e Mon Sep 17 00:00:00 2001 From: Andrey Kobrin Date: Wed, 26 Aug 2026 14:36:23 -0400 Subject: [PATCH 1/6] fix(devnet): quiet status-registry probes and bump evm-upgrade to v1.20.2 - Add lookupStatusRegistryMnemonic, a silent registry lookup for probing infrastructure-key candidates that legitimately don't exist on a host (e.g. governance_key on a secondary validator), so probes no longer emit spurious WARN logs; readStatusRegistryMnemonic keeps warning for keys that are expected to be registered - Silence updateStatusRegistryAddress for untracked accounts: generated pre-evm-* fixtures live only in accounts-devnet.json, so a missing registry entry is the normal case - Remove unused appendStatusRegistryAccount - Fix Height format verb (%d -> %s) in supernode migration verification - Bump devnet-evm-upgrade target from v1.20.1 to v1.20.2 (via new devnet-upgrade-1202) - Add unit tests for status registry lookup helpers Co-Authored-By: Claude Fable 5 --- Makefile.devnet | 6 +- .../tests/evmigration/migrate_validators.go | 2 +- devnet/tests/evmigration/prepare.go | 16 ++- devnet/tests/evmigration/status_registry.go | 58 +++------ .../tests/evmigration/status_registry_test.go | 116 ++++++++++++++++++ 5 files changed, 149 insertions(+), 49 deletions(-) create mode 100644 devnet/tests/evmigration/status_registry_test.go diff --git a/Makefile.devnet b/Makefile.devnet index 2f193d4c..1692255b 100644 --- a/Makefile.devnet +++ b/Makefile.devnet @@ -662,7 +662,7 @@ devnet-update-scripts: .PHONY: devnet-new-1120 .PHONY: devnet-upgrade-1110 devnet-upgrade-1111 devnet-upgrade-1120 devnet-upgrade-1201 -.PHONY: devnet-evm-upgrade +.PHONY: devnet-evm-upgrade devnet-upgrade-1202 # Upgrade a running devnet to a pre-downloaded lumera version. # Expects devnet/bin-/ to already contain the binaries. @@ -726,7 +726,7 @@ devnet-evm-upgrade: @echo "Logging to $(DEVNET_EVM_UPGRADE_LOG)" @bash -c 'set -euo pipefail; { \ BASE_VERSION=v1.12.0; \ - EVM_VERSION=v1.20.1; \ + EVM_VERSION=v1.20.2; \ echo "==> Stage: install $$BASE_VERSION devnet"; \ if ! $(MAKE) devnet-down; then \ echo "ERROR: stage install $$BASE_VERSION devnet failed during devnet-down" >&2; \ @@ -768,7 +768,7 @@ devnet-evm-upgrade: exit 1; \ fi; \ echo "==> Stage: upgrade to $$EVM_VERSION"; \ - if ! $(MAKE) devnet-upgrade-1201; then \ + if ! $(MAKE) devnet-upgrade-1202; then \ echo "ERROR: stage upgrade to $$EVM_VERSION failed" >&2; \ exit 1; \ fi; \ diff --git a/devnet/tests/evmigration/migrate_validators.go b/devnet/tests/evmigration/migrate_validators.go index b25e9d30..582c1988 100644 --- a/devnet/tests/evmigration/migrate_validators.go +++ b/devnet/tests/evmigration/migrate_validators.go @@ -736,7 +736,7 @@ func verifySupernodeMigration( for i, preHist := range preSN.PrevSupernodeAccounts { postHist := postSN.PrevSupernodeAccounts[i] if postHist.Account != preHist.Account || postHist.Height != preHist.Height { - return fmt.Errorf("PrevSupernodeAccounts[%d] changed: expected account=%s height=%d got account=%s height=%d", + return fmt.Errorf("PrevSupernodeAccounts[%d] changed: expected account=%s height=%s got account=%s height=%s", i, preHist.Account, preHist.Height, postHist.Account, postHist.Height) } } diff --git a/devnet/tests/evmigration/prepare.go b/devnet/tests/evmigration/prepare.go index d949961b..dd847f03 100644 --- a/devnet/tests/evmigration/prepare.go +++ b/devnet/tests/evmigration/prepare.go @@ -1303,18 +1303,24 @@ const infrastructureCandidateReadyTimeout = 90 * time.Second // exist on this host (e.g. governance_key on a secondary validator) it just // returns false after the first quick check without sleeping. func waitForInfrastructureKeyReady(name string, timeout time.Duration) bool { - if keyExists(name) && readStatusRegistryMnemonic(name) != "" { + // Probe with the silent lookup: absence is the expected outcome for + // candidates that don't apply to this host and must not log WARN. + registryMnemonic := func() string { + mnemonic, _ := lookupStatusRegistryMnemonic(name) + return mnemonic + } + if keyExists(name) && registryMnemonic() != "" { return true } // If neither the keyring nor the registry knows about this name at all, // there's nothing to wait for — it's a candidate that doesn't apply to // this host (e.g. governance_key on a secondary validator). - if !keyExists(name) && readStatusRegistryMnemonic(name) == "" { + if !keyExists(name) && registryMnemonic() == "" { return false } deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - if keyExists(name) && readStatusRegistryMnemonic(name) != "" { + if keyExists(name) && registryMnemonic() != "" { return true } time.Sleep(3 * time.Second) @@ -1344,8 +1350,8 @@ func recordInfrastructureLegacyAccounts(af *AccountsFile, existingByName map[str if _, ok := existingByName[addr]; ok { continue } - mnemonic := readStatusRegistryMnemonic(name) - if mnemonic == "" { + mnemonic, found := lookupStatusRegistryMnemonic(name) + if !found || mnemonic == "" { log.Printf(" WARN: %s has no mnemonic in status registry; skipping", name) continue } diff --git a/devnet/tests/evmigration/status_registry.go b/devnet/tests/evmigration/status_registry.go index 67fdf81d..48c36d2e 100644 --- a/devnet/tests/evmigration/status_registry.go +++ b/devnet/tests/evmigration/status_registry.go @@ -30,56 +30,32 @@ func loadStatusRegistryAccounts() ([]statusRegistryAccount, error) { return accounts, nil } -func readStatusRegistryMnemonic(name string) string { +// lookupStatusRegistryMnemonic reports whether `name` is tracked in the +// status registry, without logging when it isn't. Absence is a normal outcome +// when probing infrastructure-key candidates that don't apply to this host +// (e.g. governance_key on a secondary validator). +func lookupStatusRegistryMnemonic(name string) (string, bool) { accounts, err := loadStatusRegistryAccounts() if err != nil { log.Printf(" WARN: cannot read account registry %s: %v", statusRegistryFile(), err) - return "" + return "", false } for _, account := range accounts { if account.Name == name { - return strings.TrimSpace(account.Mnemonic) + return strings.TrimSpace(account.Mnemonic), true } } - log.Printf(" WARN: account %q not found in status registry %s", name, statusRegistryFile()) - return "" + return "", false } -// appendStatusRegistryAccount adds a {name, address, mnemonic} entry to the -// shared status registry if it isn't already present. Idempotent by name. -func appendStatusRegistryAccount(name, address, mnemonic string) { - registryFile := statusRegistryFile() - data, err := os.ReadFile(registryFile) - if err != nil { - log.Printf(" WARN: cannot read account registry %s: %v", registryFile, err) - return - } - var accounts []map[string]any - if err := json.Unmarshal(data, &accounts); err != nil { - log.Printf(" WARN: cannot parse account registry %s: %v", registryFile, err) - return - } - for _, account := range accounts { - if fmtName, _ := account["name"].(string); fmtName == name { - return - } - } - accounts = append(accounts, map[string]any{ - "name": name, - "address": address, - "mnemonic": mnemonic, - }) - encoded, err := json.MarshalIndent(accounts, "", " ") - if err != nil { - log.Printf(" WARN: cannot encode updated account registry %s: %v", registryFile, err) - return - } - encoded = append(encoded, '\n') - if err := os.WriteFile(registryFile, encoded, 0o644); err != nil { - log.Printf(" WARN: failed to append to account registry %s: %v", registryFile, err) - return +// readStatusRegistryMnemonic is the lookup for accounts that are expected to +// be registered (validator keys); it warns when the entry is missing. +func readStatusRegistryMnemonic(name string) string { + mnemonic, found := lookupStatusRegistryMnemonic(name) + if !found { + log.Printf(" WARN: account %q not found in status registry %s", name, statusRegistryFile()) } - log.Printf(" appended %s to account registry %s", name, registryFile) + return mnemonic } func updateStatusRegistryAddress(name, newAddr string) { @@ -105,7 +81,9 @@ func updateStatusRegistryAddress(name, newAddr string) { } } if !updated { - log.Printf(" WARN: account %q not found in status registry %s", name, registryFile) + // Not tracked: the registry only holds infrastructure keys (validator, + // governance, funders); generated pre-evm-* fixtures live solely in + // accounts-devnet.json, so skipping them silently is the normal case. return } diff --git a/devnet/tests/evmigration/status_registry_test.go b/devnet/tests/evmigration/status_registry_test.go new file mode 100644 index 00000000..d795aa9c --- /dev/null +++ b/devnet/tests/evmigration/status_registry_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "bytes" + "encoding/json" + "log" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTestStatusRegistry points *flagFile at a temp accounts file so +// statusRegistryFile() resolves to /accounts.json, then writes the given +// entries there. Restores the flag on cleanup. +func writeTestStatusRegistry(t *testing.T, accounts []statusRegistryAccount) string { + t.Helper() + dir := t.TempDir() + prev := *flagFile + *flagFile = filepath.Join(dir, "accounts-devnet.json") + t.Cleanup(func() { *flagFile = prev }) + + registryFile := filepath.Join(dir, "accounts.json") + data, err := json.Marshal(accounts) + if err != nil { + t.Fatalf("marshal registry: %v", err) + } + if err := os.WriteFile(registryFile, data, 0o644); err != nil { + t.Fatalf("write registry: %v", err) + } + return registryFile +} + +// captureLog redirects the standard logger to a buffer for the duration of +// the test and returns it. +func captureLog(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + return &buf +} + +func TestUpdateStatusRegistryAddressUpdatesTrackedAccount(t *testing.T) { + registryFile := writeTestStatusRegistry(t, []statusRegistryAccount{ + {Name: "governance_key", Address: "lumera1old", Mnemonic: "m"}, + }) + + updateStatusRegistryAddress("governance_key", "lumera1new") + + data, err := os.ReadFile(registryFile) + if err != nil { + t.Fatalf("read registry: %v", err) + } + var accounts []statusRegistryAccount + if err := json.Unmarshal(data, &accounts); err != nil { + t.Fatalf("parse registry: %v", err) + } + if len(accounts) != 1 || accounts[0].Address != "lumera1new" { + t.Fatalf("expected governance_key address updated to lumera1new, got %+v", accounts) + } +} + +// Generated pre-evm-* fixtures are tracked in accounts-devnet.json, never in +// the per-host status registry; skipping them must not spam WARN logs. +func TestUpdateStatusRegistryAddressSilentlySkipsUntrackedAccount(t *testing.T) { + registryFile := writeTestStatusRegistry(t, []statusRegistryAccount{ + {Name: "governance_key", Address: "lumera1old", Mnemonic: "m"}, + }) + before, err := os.ReadFile(registryFile) + if err != nil { + t.Fatalf("read registry: %v", err) + } + buf := captureLog(t) + + updateStatusRegistryAddress("pre-evm-val5-003", "lumera1new") + + if out := buf.String(); strings.Contains(out, "WARN") { + t.Fatalf("expected no WARN for untracked account, got log output: %q", out) + } + after, err := os.ReadFile(registryFile) + if err != nil { + t.Fatalf("read registry: %v", err) + } + if !bytes.Equal(before, after) { + t.Fatalf("registry file changed for untracked account:\nbefore: %s\nafter: %s", before, after) + } +} + +func TestLookupStatusRegistryMnemonicFound(t *testing.T) { + writeTestStatusRegistry(t, []statusRegistryAccount{ + {Name: "sncli-account", Address: "lumera1abc", Mnemonic: " word1 word2 "}, + }) + + mnemonic, found := lookupStatusRegistryMnemonic("sncli-account") + if !found || mnemonic != "word1 word2" { + t.Fatalf("lookupStatusRegistryMnemonic = (%q, %v), want (\"word1 word2\", true)", mnemonic, found) + } +} + +// Infrastructure-key probes check hosts that legitimately don't have the key +// (e.g. governance_key on a secondary validator); the lookup must stay silent. +func TestLookupStatusRegistryMnemonicNotFoundIsSilent(t *testing.T) { + writeTestStatusRegistry(t, []statusRegistryAccount{ + {Name: "supernova_validator_5_key", Address: "lumera1abc", Mnemonic: "m"}, + }) + buf := captureLog(t) + + mnemonic, found := lookupStatusRegistryMnemonic("governance_key") + if found || mnemonic != "" { + t.Fatalf("lookupStatusRegistryMnemonic = (%q, %v), want (\"\", false)", mnemonic, found) + } + if out := buf.String(); strings.Contains(out, "WARN") { + t.Fatalf("expected no WARN for absent probe candidate, got log output: %q", out) + } +} From 545f7c69f31b0c196cf76a8ba362865eddaa2360 Mon Sep 17 00:00:00 2001 From: Andrey Kobrin Date: Mon, 31 Aug 2026 17:20:53 -0400 Subject: [PATCH 2/6] fix(ci): address PR 209 review and test flakes --- Makefile.devnet | 7 ++-- cmd/lumera/cmd/commands.go | 35 +++++++++++++--- cmd/lumera/cmd/commands_test.go | 44 ++++++++++++++++++++ devnet/scripts/upgrade.sh | 24 +++++++---- tests/integration/evmtest/tx_helpers.go | 43 +++++++++++++------ tests/integration/evmtest/tx_helpers_test.go | 34 +++++++++++++++ tests/scripts/devnet-makefile.bats | 7 ++++ tests/systemtests/system.go | 8 ++-- 8 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 cmd/lumera/cmd/commands_test.go create mode 100644 tests/integration/evmtest/tx_helpers_test.go diff --git a/Makefile.devnet b/Makefile.devnet index 1692255b..505a6d0e 100644 --- a/Makefile.devnet +++ b/Makefile.devnet @@ -701,8 +701,9 @@ devnet-upgrade-1201: @$(MAKE) devnet-refresh-bin @cd devnet/scripts && ./upgrade.sh v1.20.1 auto-height ../bin -# v1.20.2 — same locally-built-binary pattern as devnet-upgrade-1201, because -# v1.20.2 has no published release to pre-download. Drives the coordinated +# v1.20.2 — same locally-built-binary pattern as devnet-upgrade-1201. The +# on-chain plan remains v1.20.2, but the superseding binary must report v1.20.3. +# Drives the coordinated # governance halt + binary swap, which is MANDATORY for this upgrade: v1.20.2 # changes evmigration DeliverTx outcomes (PrevSupernodeAccounts append vs # rewrite, canonical ownership resolution, Everlight SNDistState move), so a @@ -714,7 +715,7 @@ devnet-upgrade-1201: # mainnet-shaped 1.12.0 -> 1.20.2 full EVM bring-up + add-only store mount devnet-upgrade-1202: @$(MAKE) devnet-refresh-bin - @cd devnet/scripts && ./upgrade.sh v1.20.2 auto-height ../bin + @cd devnet/scripts && ./upgrade.sh v1.20.2 auto-height ../bin v1.20.3 devnet-new-1120: @$(MAKE) devnet-new-version VERSION=v1.12.0 diff --git a/cmd/lumera/cmd/commands.go b/cmd/lumera/cmd/commands.go index 0cdc367b..5fa2710b 100644 --- a/cmd/lumera/cmd/commands.go +++ b/cmd/lumera/cmd/commands.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "strconv" tmcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" cmttypes "github.com/cometbft/cometbft/types" @@ -134,7 +135,7 @@ func wrapJSONRPCAliasStartPreRun(startCmd *cobra.Command) { return nil } - internalAddr, err := reserveLoopbackAddr() + internalAddr, err := reserveLoopbackAddr(publicAddr) if err != nil { return err } @@ -146,16 +147,40 @@ func wrapJSONRPCAliasStartPreRun(startCmd *cobra.Command) { } } -func reserveLoopbackAddr() (string, error) { - ln, err := net.Listen("tcp", "127.0.0.1:0") +func reserveLoopbackAddr(publicAddr string) (string, error) { + internalAddr, err := loopbackAddrForPublic(publicAddr) if err != nil { return "", err } - addr := ln.Addr().String() + + // Verify the derived address is currently available. Closing it before the + // native JSON-RPC server binds still leaves a small external-process race, + // but unlike port 0 the deterministic mapping prevents sibling lumerad + // processes with distinct public ports from selecting the same upstream. + ln, err := net.Listen("tcp", internalAddr) + if err != nil { + return "", fmt.Errorf("reserve internal JSON-RPC address %s: %w", internalAddr, err) + } if closeErr := ln.Close(); closeErr != nil { return "", closeErr } - return addr, nil + return internalAddr, nil +} + +func loopbackAddrForPublic(publicAddr string) (string, error) { + _, portText, err := net.SplitHostPort(publicAddr) + if err != nil { + return "", fmt.Errorf("parse public JSON-RPC address %q: %w", publicAddr, err) + } + publicPort, err := strconv.Atoi(portText) + if err != nil || publicPort < 1 || publicPort > 65535 { + return "", fmt.Errorf("invalid public JSON-RPC port %q", portText) + } + + // Rotate the valid TCP port range by 32768 positions. This is a one-to-one + // mapping, so distinct public ports always yield distinct internal ports. + internalPort := ((publicPort + 32767) % 65535) + 1 + return net.JoinHostPort("127.0.0.1", strconv.Itoa(internalPort)), nil } func addModuleInitFlags(startCmd *cobra.Command) { diff --git a/cmd/lumera/cmd/commands_test.go b/cmd/lumera/cmd/commands_test.go new file mode 100644 index 00000000..0795a4c7 --- /dev/null +++ b/cmd/lumera/cmd/commands_test.go @@ -0,0 +1,44 @@ +package cmd + +import "testing" + +func TestLoopbackAddrForPublicIsStableAndUnique(t *testing.T) { + t.Parallel() + + tests := []struct { + public string + want string + }{ + {public: "127.0.0.1:8545", want: "127.0.0.1:41313"}, + {public: "0.0.0.0:8645", want: "127.0.0.1:41413"}, + {public: "[::]:8745", want: "127.0.0.1:41513"}, + {public: "localhost:32768", want: "127.0.0.1:1"}, + } + + seen := make(map[string]string, len(tests)) + for _, tc := range tests { + t.Run(tc.public, func(t *testing.T) { + got, err := loopbackAddrForPublic(tc.public) + if err != nil { + t.Fatalf("loopbackAddrForPublic(%q): %v", tc.public, err) + } + if got != tc.want { + t.Fatalf("loopbackAddrForPublic(%q) = %q, want %q", tc.public, got, tc.want) + } + if previous, exists := seen[got]; exists { + t.Fatalf("public addresses %q and %q mapped to the same internal address %q", previous, tc.public, got) + } + seen[got] = tc.public + }) + } +} + +func TestLoopbackAddrForPublicRejectsInvalidAddress(t *testing.T) { + t.Parallel() + + for _, publicAddr := range []string{"", "127.0.0.1", "127.0.0.1:0", "127.0.0.1:65536"} { + if _, err := loopbackAddrForPublic(publicAddr); err == nil { + t.Fatalf("loopbackAddrForPublic(%q) unexpectedly succeeded", publicAddr) + } + } +} diff --git a/devnet/scripts/upgrade.sh b/devnet/scripts/upgrade.sh index 74b18e97..df431c48 100755 --- a/devnet/scripts/upgrade.sh +++ b/devnet/scripts/upgrade.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -ne 3 ]]; then - echo "Usage: $0 " +if [[ $# -lt 3 || $# -gt 4 ]]; then + echo "Usage: $0 [expected-binary-version]" exit 1 fi RELEASE_NAME="$1" REQUESTED_HEIGHT="$2" BINARIES_DIR="$3" +EXPECTED_BINARY_RELEASE_NAME="${4:-${RELEASE_NAME}}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null @@ -82,14 +83,19 @@ duration_to_seconds() { RUNNING_VERSION="$(docker compose -f "${COMPOSE_FILE}" exec -T "${SERVICE}" \ lumerad version 2>/dev/null | head -n 1 | tr -d '\r' || true)" RUNNING_VERSION="$(normalize_version "${RUNNING_VERSION}")" -EXPECTED_VERSION="$(normalize_version "${RELEASE_NAME}")" +EXPECTED_BINARY_VERSION="$(normalize_version "${EXPECTED_BINARY_RELEASE_NAME}")" -if [[ -n "${RUNNING_VERSION}" && "${RUNNING_VERSION}" == "${EXPECTED_VERSION}" ]]; then - echo "Node is already running version ${RUNNING_VERSION}. Upgrade to ${RELEASE_NAME} already complete." +if [[ -z "${EXPECTED_BINARY_VERSION}" ]]; then + echo "Expected binary version is empty or invalid: ${EXPECTED_BINARY_RELEASE_NAME}" >&2 + exit 1 +fi + +if [[ -n "${RUNNING_VERSION}" && "${RUNNING_VERSION}" == "${EXPECTED_BINARY_VERSION}" ]]; then + echo "Node is already running version ${RUNNING_VERSION}. Upgrade plan ${RELEASE_NAME} already complete." exit 0 fi -if [[ -n "${RUNNING_VERSION}" ]] && versions_match "${EXPECTED_VERSION}" "${RUNNING_VERSION}"; then - echo "Node is already running compatible version ${RUNNING_VERSION}. Upgrade to ${RELEASE_NAME} already complete." +if [[ -n "${RUNNING_VERSION}" ]] && versions_match "${EXPECTED_BINARY_VERSION}" "${RUNNING_VERSION}"; then + echo "Node is already running compatible version ${RUNNING_VERSION}. Upgrade plan ${RELEASE_NAME} already complete." exit 0 fi @@ -102,7 +108,7 @@ if [[ "${REQUESTED_HEIGHT}" == "auto-height" ]]; then # Chain is not responding — check if it halted for our upgrade if detect_upgrade_halt; then echo "Chain is already halted for ${RELEASE_NAME} upgrade. Skipping to binary upgrade..." - "${SCRIPT_DIR}/upgrade-binaries.sh" "${BINARIES_DIR}" "${RELEASE_NAME}" + "${SCRIPT_DIR}/upgrade-binaries.sh" "${BINARIES_DIR}" "${EXPECTED_BINARY_RELEASE_NAME}" echo "Upgrade to ${RELEASE_NAME} initiated successfully." exit 0 fi @@ -266,6 +272,6 @@ if ! detect_upgrade_halt; then fi echo "Upgrading binaries from ${BINARIES_DIR}..." -"${SCRIPT_DIR}/upgrade-binaries.sh" "${BINARIES_DIR}" "${RELEASE_NAME}" +"${SCRIPT_DIR}/upgrade-binaries.sh" "${BINARIES_DIR}" "${EXPECTED_BINARY_RELEASE_NAME}" echo "Upgrade to ${RELEASE_NAME} initiated successfully." diff --git a/tests/integration/evmtest/tx_helpers.go b/tests/integration/evmtest/tx_helpers.go index 767c30a7..16a475a0 100644 --- a/tests/integration/evmtest/tx_helpers.go +++ b/tests/integration/evmtest/tx_helpers.go @@ -34,20 +34,39 @@ func sendOneLegacyTx(t *testing.T, rpcURL string, keyInfo testaccounts.TestKeyIn fromAddr := testaccounts.MustAccountAddressFromTestKeyInfo(t, keyInfo) privateKey := mustDerivePrivateKey(t, keyInfo.Mnemonic) - - nonce := mustGetPendingNonceWithRetry(t, rpcURL, fromAddr.Hex(), 20*time.Second) - gasPrice := mustGetGasPriceWithRetry(t, rpcURL, 20*time.Second) toAddr := fromAddr + deadline := time.Now().Add(10 * time.Second) + + for { + nonce := mustGetPendingNonceWithRetry(t, rpcURL, fromAddr.Hex(), 20*time.Second) + gasPrice := mustGetGasPriceWithRetry(t, rpcURL, 20*time.Second) + txHash, err := sendLegacyTxWithParamsResult(rpcURL, legacyTxParams{ + PrivateKey: privateKey, + Nonce: nonce, + To: &toAddr, + Value: big.NewInt(1), + Gas: 21_000, + GasPrice: gasPrice, + Data: nil, + }) + if err == nil { + return txHash + } + if !isTransientBlockGasLimitError(err) || time.Now().After(deadline) { + t.Fatalf("send legacy tx: %v", err) + } - return sendLegacyTxWithParams(t, rpcURL, legacyTxParams{ - PrivateKey: privateKey, - Nonce: nonce, - To: &toAddr, - Value: big.NewInt(1), - Gas: 21_000, - GasPrice: gasPrice, - Data: nil, - }) + // At startup the JSON-RPC mempool can briefly observe the previous + // block's exhausted/zero remaining gas before the next block opens. A + // plain 21k transfer is always below this fixture's 25M block limit, so + // retry only this specific transient rejection across a block boundary. + t.Logf("retrying legacy tx after transient block gas-limit rejection: %v", err) + time.Sleep(250 * time.Millisecond) + } +} + +func isTransientBlockGasLimitError(err error) bool { + return err != nil && strings.Contains(err.Error(), "exceeds block gas limit") } // sendOneCosmosBankTx broadcasts a simple bank MsgSend transaction and returns tx hash. diff --git a/tests/integration/evmtest/tx_helpers_test.go b/tests/integration/evmtest/tx_helpers_test.go new file mode 100644 index 00000000..d0be8804 --- /dev/null +++ b/tests/integration/evmtest/tx_helpers_test.go @@ -0,0 +1,34 @@ +//go:build integration +// +build integration + +package evmtest + +import ( + "errors" + "fmt" + "testing" +) + +func TestIsTransientBlockGasLimitError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "unrelated rejection", err: errors.New("nonce too low"), want: false}, + {name: "direct rejection", err: errors.New("exceeds block gas limit"), want: true}, + {name: "wrapped RPC rejection", err: fmt.Errorf("broadcast failed: %w", errors.New("exceeds block gas limit: internal")), want: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isTransientBlockGasLimitError(tc.err); got != tc.want { + t.Fatalf("isTransientBlockGasLimitError(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} diff --git a/tests/scripts/devnet-makefile.bats b/tests/scripts/devnet-makefile.bats index fde74ceb..aa6ed80e 100644 --- a/tests/scripts/devnet-makefile.bats +++ b/tests/scripts/devnet-makefile.bats @@ -54,3 +54,10 @@ teardown() { [[ "$output" == *"--remove-orphans"* ]] [[ "$output" != *"ssh example-devnet"*"go "* ]] } + +@test "v1.20.2 upgrade plan installs the superseding v1.20.3 binary" { + run make -C "$REPO_ROOT" -n devnet-upgrade-1202 MAKE=/bin/true + + [ "$status" -eq 0 ] + [[ "$output" == *"./upgrade.sh v1.20.2 auto-height ../bin v1.20.3"* ]] +} diff --git a/tests/systemtests/system.go b/tests/systemtests/system.go index dfd3a093..80833238 100644 --- a/tests/systemtests/system.go +++ b/tests/systemtests/system.go @@ -656,8 +656,10 @@ func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) { }(pid, cmd, i) }) - // Wait in background and close the channel when all nodes are done - go func() { + // Drain node process results during test cleanup. Reporting through t from a + // detached goroutine can panic if a node exits after the test has completed. + t.Cleanup(func() { + s.StopChain() wg.Wait() close(errChan) @@ -666,7 +668,7 @@ func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) { t.Errorf("%v", err) } } - }() + }) } func (s *SystemUnderTest) withEachNodeHome(cb func(i int, home string)) { From 2942b487f0faa677299fd279c07892d47c0f99cf Mon Sep 17 00:00:00 2001 From: Andrey Kobrin Date: Mon, 31 Aug 2026 17:41:04 -0400 Subject: [PATCH 3/6] fix(jsonrpc): keep derived upstream ports unprivileged --- cmd/lumera/cmd/commands.go | 8 +++++--- cmd/lumera/cmd/commands_test.go | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/lumera/cmd/commands.go b/cmd/lumera/cmd/commands.go index 5fa2710b..c6a723c8 100644 --- a/cmd/lumera/cmd/commands.go +++ b/cmd/lumera/cmd/commands.go @@ -177,9 +177,11 @@ func loopbackAddrForPublic(publicAddr string) (string, error) { return "", fmt.Errorf("invalid public JSON-RPC port %q", portText) } - // Rotate the valid TCP port range by 32768 positions. This is a one-to-one - // mapping, so distinct public ports always yield distinct internal ports. - internalPort := ((publicPort + 32767) % 65535) + 1 + // Rotate within the unprivileged TCP port range. Integration fixtures use + // ephemeral public ports, so rotating across the full 1..65535 range could + // map them below 1024 and fail for non-root processes. + const unprivilegedPortCount = 65535 - 1024 + 1 + internalPort := 1024 + ((publicPort - 1 + 32768) % unprivilegedPortCount) return net.JoinHostPort("127.0.0.1", strconv.Itoa(internalPort)), nil } diff --git a/cmd/lumera/cmd/commands_test.go b/cmd/lumera/cmd/commands_test.go index 0795a4c7..ebcf916d 100644 --- a/cmd/lumera/cmd/commands_test.go +++ b/cmd/lumera/cmd/commands_test.go @@ -9,10 +9,11 @@ func TestLoopbackAddrForPublicIsStableAndUnique(t *testing.T) { public string want string }{ - {public: "127.0.0.1:8545", want: "127.0.0.1:41313"}, - {public: "0.0.0.0:8645", want: "127.0.0.1:41413"}, - {public: "[::]:8745", want: "127.0.0.1:41513"}, - {public: "localhost:32768", want: "127.0.0.1:1"}, + {public: "127.0.0.1:8545", want: "127.0.0.1:42336"}, + {public: "0.0.0.0:8645", want: "127.0.0.1:42436"}, + {public: "[::]:8745", want: "127.0.0.1:42536"}, + {public: "localhost:32768", want: "127.0.0.1:2047"}, + {public: "127.0.0.1:32863", want: "127.0.0.1:2142"}, } seen := make(map[string]string, len(tests)) From 69f8c15fbc993168a28c7e7f6e2335ae2a0c16ca Mon Sep 17 00:00:00 2001 From: Andrey Kobrin Date: Mon, 31 Aug 2026 18:13:33 -0400 Subject: [PATCH 4/6] fix(jsonrpc): probe unprivileged upstream fallbacks --- cmd/lumera/cmd/commands.go | 67 +++++++++++++++++++++++++++------ cmd/lumera/cmd/commands_test.go | 58 +++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 12 deletions(-) diff --git a/cmd/lumera/cmd/commands.go b/cmd/lumera/cmd/commands.go index c6a723c8..508d1bef 100644 --- a/cmd/lumera/cmd/commands.go +++ b/cmd/lumera/cmd/commands.go @@ -7,6 +7,7 @@ import ( "io" "net" "strconv" + "syscall" tmcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" cmttypes "github.com/cometbft/cometbft/types" @@ -153,20 +154,64 @@ func reserveLoopbackAddr(publicAddr string) (string, error) { return "", err } - // Verify the derived address is currently available. Closing it before the - // native JSON-RPC server binds still leaves a small external-process race, - // but unlike port 0 the deterministic mapping prevents sibling lumerad - // processes with distinct public ports from selecting the same upstream. - ln, err := net.Listen("tcp", internalAddr) + _, primaryPortText, err := net.SplitHostPort(internalAddr) if err != nil { - return "", fmt.Errorf("reserve internal JSON-RPC address %s: %w", internalAddr, err) + return "", err + } + primaryPort, err := strconv.Atoi(primaryPortText) + if err != nil { + return "", err + } + _, publicPortText, err := net.SplitHostPort(publicAddr) + if err != nil { + return "", err } - if closeErr := ln.Close(); closeErr != nil { - return "", closeErr + publicPort, err := strconv.Atoi(publicPortText) + if err != nil { + return "", err } - return internalAddr, nil + + // Verify a deterministic candidate is currently available. Closing it + // before the native JSON-RPC server binds still leaves a small + // external-process race, but the deterministic primary prevents sibling + // lumerad processes with distinct public ports from selecting the same + // upstream. If another service owns that primary, walk a deterministic + // permutation of the unprivileged range instead of making an otherwise + // valid public listener unusable. The relatively prime step keeps nearby + // public ports from immediately falling back onto each other's primaries. + const fallbackProbeStep = 7919 + for attempt := 0; attempt < unprivilegedPortCount; attempt++ { + candidatePort := firstUnprivilegedPort + + ((primaryPort-firstUnprivilegedPort)+(attempt*fallbackProbeStep))%unprivilegedPortCount + // The alias proxy needs the operator-configured public port later in + // startup, so never consume it as the native server's upstream. + if candidatePort == publicPort { + continue + } + + candidateAddr := net.JoinHostPort("127.0.0.1", strconv.Itoa(candidatePort)) + ln, listenErr := net.Listen("tcp", candidateAddr) + if listenErr != nil { + if errors.Is(listenErr, syscall.EADDRINUSE) { + continue + } + return "", fmt.Errorf("reserve internal JSON-RPC address %s: %w", candidateAddr, listenErr) + } + if closeErr := ln.Close(); closeErr != nil { + return "", closeErr + } + return candidateAddr, nil + } + + return "", fmt.Errorf("no unprivileged internal JSON-RPC port available for %s", publicAddr) } +const ( + firstUnprivilegedPort = 1024 + lastUnprivilegedPort = 65535 + unprivilegedPortCount = lastUnprivilegedPort - firstUnprivilegedPort + 1 +) + func loopbackAddrForPublic(publicAddr string) (string, error) { _, portText, err := net.SplitHostPort(publicAddr) if err != nil { @@ -180,8 +225,8 @@ func loopbackAddrForPublic(publicAddr string) (string, error) { // Rotate within the unprivileged TCP port range. Integration fixtures use // ephemeral public ports, so rotating across the full 1..65535 range could // map them below 1024 and fail for non-root processes. - const unprivilegedPortCount = 65535 - 1024 + 1 - internalPort := 1024 + ((publicPort - 1 + 32768) % unprivilegedPortCount) + internalPort := firstUnprivilegedPort + + ((publicPort - 1 + 32768) % unprivilegedPortCount) return net.JoinHostPort("127.0.0.1", strconv.Itoa(internalPort)), nil } diff --git a/cmd/lumera/cmd/commands_test.go b/cmd/lumera/cmd/commands_test.go index ebcf916d..8b5af2a4 100644 --- a/cmd/lumera/cmd/commands_test.go +++ b/cmd/lumera/cmd/commands_test.go @@ -1,6 +1,10 @@ package cmd -import "testing" +import ( + "net" + "strconv" + "testing" +) func TestLoopbackAddrForPublicIsStableAndUnique(t *testing.T) { t.Parallel() @@ -43,3 +47,55 @@ func TestLoopbackAddrForPublicRejectsInvalidAddress(t *testing.T) { } } } + +func TestReserveLoopbackAddrFallsBackWhenPrimaryIsOccupied(t *testing.T) { + occupied, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on occupied primary: %v", err) + } + t.Cleanup(func() { _ = occupied.Close() }) + + occupiedPort := occupied.Addr().(*net.TCPAddr).Port + publicAddr := publicAddrForInternalPort(t, occupiedPort) + got, err := reserveLoopbackAddr(publicAddr) + if err != nil { + t.Fatalf("reserveLoopbackAddr(%q): %v", publicAddr, err) + } + if got == occupied.Addr().String() { + t.Fatalf("reserveLoopbackAddr(%q) returned occupied primary %q", publicAddr, got) + } + + _, portText, err := net.SplitHostPort(got) + if err != nil { + t.Fatalf("parse fallback address %q: %v", got, err) + } + port, err := strconv.Atoi(portText) + if err != nil || port < firstUnprivilegedPort || port > lastUnprivilegedPort { + t.Fatalf("fallback address %q is not unprivileged", got) + } + + probe, err := net.Listen("tcp", got) + if err != nil { + t.Fatalf("fallback address %q is not available: %v", got, err) + } + _ = probe.Close() +} + +func publicAddrForInternalPort(t *testing.T, internalPort int) string { + t.Helper() + + want := net.JoinHostPort("127.0.0.1", strconv.Itoa(internalPort)) + for publicPort := firstUnprivilegedPort; publicPort <= lastUnprivilegedPort; publicPort++ { + publicAddr := net.JoinHostPort("127.0.0.1", strconv.Itoa(publicPort)) + got, err := loopbackAddrForPublic(publicAddr) + if err != nil { + t.Fatalf("loopbackAddrForPublic(%q): %v", publicAddr, err) + } + if got == want { + return publicAddr + } + } + + t.Fatalf("no public port maps to internal port %d", internalPort) + return "" +} From 1edc7280a366bc183ab003fa290493ca2cd5b414 Mon Sep 17 00:00:00 2001 From: Andrey Kobrin Date: Mon, 31 Aug 2026 18:17:41 -0400 Subject: [PATCH 5/6] test(cmd): serialize SDK global mutations --- cmd/lumera/cmd/config_migrate_test.go | 8 -------- cmd/lumera/cmd/root_test.go | 4 ---- 2 files changed, 12 deletions(-) diff --git a/cmd/lumera/cmd/config_migrate_test.go b/cmd/lumera/cmd/config_migrate_test.go index 8ea6b414..c52cd1e3 100644 --- a/cmd/lumera/cmd/config_migrate_test.go +++ b/cmd/lumera/cmd/config_migrate_test.go @@ -112,8 +112,6 @@ func TestNeedsConfigMigration_DisabledMempool(t *testing.T) { // start with a legacy pre-EVM app.toml, run the migrator, and confirm both // the disk file and in-memory Viper contain the correct EVM config. func TestMigrateAppConfig_LegacyTomlOnDisk(t *testing.T) { - t.Parallel() - // Create a temp directory with a minimal legacy app.toml (no EVM sections). tmpDir := t.TempDir() configDir := filepath.Join(tmpDir, "config") @@ -193,8 +191,6 @@ max-txs = 3000 } func TestMigrateAppConfig_FullyMigratedNegativeMaxTxsTriggersRepair(t *testing.T) { - t.Parallel() - tmpDir := t.TempDir() configDir := filepath.Join(tmpDir, "config") require.NoError(t, os.MkdirAll(configDir, 0o755)) @@ -245,8 +241,6 @@ certificate-path = "" } func TestMigrateAppConfig_LegacyNegativeMaxTxsUsesNetworkDefault(t *testing.T) { - t.Parallel() - testCases := []struct { name string chainID string @@ -262,8 +256,6 @@ func TestMigrateAppConfig_LegacyNegativeMaxTxsUsesNetworkDefault(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tmpDir := t.TempDir() configDir := filepath.Join(tmpDir, "config") require.NoError(t, os.MkdirAll(configDir, 0o755)) diff --git a/cmd/lumera/cmd/root_test.go b/cmd/lumera/cmd/root_test.go index 4f4a16ef..d6caa5a9 100644 --- a/cmd/lumera/cmd/root_test.go +++ b/cmd/lumera/cmd/root_test.go @@ -25,8 +25,6 @@ func TestNewRootCmd_DoesNotPanic(t *testing.T) { // TestNewRootCmdStartWiresEVMFlags verifies `start` command includes Cosmos EVM // server flags required by JSON-RPC and indexer startup path. func TestNewRootCmdStartWiresEVMFlags(t *testing.T) { - t.Parallel() - rootCmd := NewRootCmd() startCmd := mustFindSubcommand(t, rootCmd, "start") @@ -39,8 +37,6 @@ func TestNewRootCmdStartWiresEVMFlags(t *testing.T) { // TestNewRootCmdDefaultKeyTypeOverridden verifies recursive default overrides // set EthSecp256k1 key type across key-management and testnet commands. func TestNewRootCmdDefaultKeyTypeOverridden(t *testing.T) { - t.Parallel() - rootCmd := NewRootCmd() expectedAlgo := string(evmhd.EthSecp256k1Type) From 8e41d4666bf3da8aad986cbce2cd25c9e4d1cec4 Mon Sep 17 00:00:00 2001 From: Andrey Kobrin Date: Mon, 31 Aug 2026 18:45:18 -0400 Subject: [PATCH 6/6] fix(jsonrpc): avoid configured listener ports --- cmd/lumera/cmd/commands.go | 70 +++++++++++++++++++++++++++------ cmd/lumera/cmd/commands_test.go | 26 ++++++++++++ 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/cmd/lumera/cmd/commands.go b/cmd/lumera/cmd/commands.go index 508d1bef..2afd518f 100644 --- a/cmd/lumera/cmd/commands.go +++ b/cmd/lumera/cmd/commands.go @@ -7,6 +7,7 @@ import ( "io" "net" "strconv" + "strings" "syscall" tmcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" @@ -136,7 +137,35 @@ func wrapJSONRPCAliasStartPreRun(startCmd *cobra.Command) { return nil } - internalAddr, err := reserveLoopbackAddr(publicAddr) + excludedAddrs := []string{ + v.GetString("json-rpc.ws-address"), + v.GetString("json-rpc.metrics-address"), + v.GetString("evm.geth-metrics-address"), + v.GetString("api.address"), + v.GetString("grpc.address"), + v.GetString("lumera.json-rpc-ratelimit.proxy-address"), + } + if cometConfig := serverCtx.Config; cometConfig != nil { + excludedAddrs = append(excludedAddrs, + cometConfig.ProxyApp, + cometConfig.PrivValidatorListenAddr, + ) + if cometConfig.RPC != nil { + excludedAddrs = append(excludedAddrs, + cometConfig.RPC.ListenAddress, + cometConfig.RPC.GRPCListenAddress, + cometConfig.RPC.PprofListenAddress, + ) + } + if cometConfig.P2P != nil { + excludedAddrs = append(excludedAddrs, cometConfig.P2P.ListenAddress) + } + if cometConfig.Instrumentation != nil { + excludedAddrs = append(excludedAddrs, cometConfig.Instrumentation.PrometheusListenAddr) + } + } + + internalAddr, err := reserveLoopbackAddr(publicAddr, excludedAddrs...) if err != nil { return err } @@ -148,7 +177,7 @@ func wrapJSONRPCAliasStartPreRun(startCmd *cobra.Command) { } } -func reserveLoopbackAddr(publicAddr string) (string, error) { +func reserveLoopbackAddr(publicAddr string, excludedAddrs ...string) (string, error) { internalAddr, err := loopbackAddrForPublic(publicAddr) if err != nil { return "", err @@ -162,13 +191,15 @@ func reserveLoopbackAddr(publicAddr string) (string, error) { if err != nil { return "", err } - _, publicPortText, err := net.SplitHostPort(publicAddr) - if err != nil { - return "", err + + excludedPorts := make(map[int]struct{}, len(excludedAddrs)+1) + if publicPort, ok := portFromListenAddr(publicAddr); ok { + excludedPorts[publicPort] = struct{}{} } - publicPort, err := strconv.Atoi(publicPortText) - if err != nil { - return "", err + for _, addr := range excludedAddrs { + if port, ok := portFromListenAddr(addr); ok { + excludedPorts[port] = struct{}{} + } } // Verify a deterministic candidate is currently available. Closing it @@ -183,9 +214,10 @@ func reserveLoopbackAddr(publicAddr string) (string, error) { for attempt := 0; attempt < unprivilegedPortCount; attempt++ { candidatePort := firstUnprivilegedPort + ((primaryPort-firstUnprivilegedPort)+(attempt*fallbackProbeStep))%unprivilegedPortCount - // The alias proxy needs the operator-configured public port later in - // startup, so never consume it as the native server's upstream. - if candidatePort == publicPort { + // The alias proxy and the daemon's other servers bind later in startup, + // so never consume one of their configured ports as the native HTTP + // server's upstream. + if _, excluded := excludedPorts[candidatePort]; excluded { continue } @@ -206,6 +238,22 @@ func reserveLoopbackAddr(publicAddr string) (string, error) { return "", fmt.Errorf("no unprivileged internal JSON-RPC port available for %s", publicAddr) } +func portFromListenAddr(addr string) (int, bool) { + addr = strings.TrimSpace(addr) + if _, remainder, hasScheme := strings.Cut(addr, "://"); hasScheme { + addr = remainder + } + _, portText, err := net.SplitHostPort(addr) + if err != nil { + return 0, false + } + port, err := strconv.Atoi(portText) + if err != nil || port < 1 || port > 65535 { + return 0, false + } + return port, true +} + const ( firstUnprivilegedPort = 1024 lastUnprivilegedPort = 65535 diff --git a/cmd/lumera/cmd/commands_test.go b/cmd/lumera/cmd/commands_test.go index 8b5af2a4..b193e5df 100644 --- a/cmd/lumera/cmd/commands_test.go +++ b/cmd/lumera/cmd/commands_test.go @@ -81,6 +81,32 @@ func TestReserveLoopbackAddrFallsBackWhenPrimaryIsOccupied(t *testing.T) { _ = probe.Close() } +func TestReserveLoopbackAddrSkipsLaterListenerPorts(t *testing.T) { + const publicAddr = "0.0.0.0:39267" // primary candidate is default WS port 8546 + primary, err := loopbackAddrForPublic(publicAddr) + if err != nil { + t.Fatalf("loopbackAddrForPublic(%q): %v", publicAddr, err) + } + if primary != "127.0.0.1:8546" { + t.Fatalf("test precondition: primary = %q, want default WS address", primary) + } + + for _, excludedAddr := range []string{ + "127.0.0.1:8546", + "tcp://127.0.0.1:8546", + } { + t.Run(excludedAddr, func(t *testing.T) { + got, err := reserveLoopbackAddr(publicAddr, excludedAddr) + if err != nil { + t.Fatalf("reserveLoopbackAddr(%q): %v", publicAddr, err) + } + if got == "127.0.0.1:8546" { + t.Fatalf("reserveLoopbackAddr(%q) selected excluded listener %q", publicAddr, got) + } + }) + } +} + func publicAddrForInternalPort(t *testing.T, internalPort int) string { t.Helper()