From 07a412c3d57f159fd5e06cc9b8fccb52c0b019b9 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Tue, 25 Aug 2026 17:17:11 +0200 Subject: [PATCH 01/54] =?UTF-8?q?Upgrade=20cilium/ebpf=20from=20v0.11.0=20?= =?UTF-8?q?to=20v0.18.0=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=E2=94=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgrade ebpf and dependencies to protect from CVE in cilium/ebpf commit e30ccb7bbbc3dfa7376773ab08f977fba4710e99. The vulnerability affects memory_unsafe.go which was added in v0.19.0. v0.18.0 is the highest version without this file, providing protection while maintaining compatibility with existing dependencies. Also upgrades: - containerd/cgroups: v1.0.1 -> v1.1.0 - containerd/cgroups/v3: v3.0.3 -> v3.1.3 - google/nftables: old -> v0.3.0 - Various transitive dependencies --- go.mod | 29 ++++------ go.sum | 164 +++++++++++---------------------------------------------- 2 files changed, 41 insertions(+), 152 deletions(-) diff --git a/go.mod b/go.mod index fd4e9bbac..54f1d8877 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/antchfx/xmlquery v1.3.10 github.com/atomicgo/cursor v0.0.1 - github.com/containerd/cgroups v1.0.1 + github.com/containerd/cgroups v1.1.0 github.com/containerd/cgroups/v3 v3.0.3 github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 @@ -29,7 +29,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.0 github.com/golang/mock v1.5.0 github.com/google/go-cmp v0.7.0 - github.com/google/nftables v0.0.0-20220129182606-a46119e5928d + github.com/google/nftables v0.3.0 github.com/google/uuid v1.6.0 github.com/goombaio/orderedset v0.0.0-20180925151225-8e67b20a9b77 github.com/hashicorp/go-version v1.4.0 @@ -70,13 +70,13 @@ require ( github.com/stretchr/testify v1.11.1 github.com/subosito/gotenv v1.2.0 github.com/talos-systems/go-smbios v0.1.1 - github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 - github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f + github.com/vishvananda/netlink v1.3.0 + github.com/vishvananda/netns v0.0.4 github.com/ybbus/jsonrpc v2.1.2+incompatible github.com/yookoala/realpath v1.0.0 github.com/zcalusic/sysinfo v0.0.0-20210831153053-2c6e1d254246 golang.org/x/crypto v0.53.0 - golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 + golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f golang.org/x/net v0.56.0 golang.org/x/sync v0.21.0 golang.org/x/sys v0.46.0 @@ -91,31 +91,28 @@ require ( ) require ( - github.com/BurntSushi/toml v1.3.2 // indirect github.com/antchfx/xpath v1.3.6 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cilium/ebpf v0.11.0 // indirect + github.com/cilium/ebpf v0.18.0 // indirect github.com/coreos/go-iptables v0.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/docker/go-units v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/go-openapi/jsonpointer v0.22.5 // indirect github.com/go-openapi/swag/jsonname v0.25.5 // indirect - github.com/godbus/dbus/v5 v5.0.4 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/goombaio/orderedmap v0.0.0-20180924084748-ba921b7e2419 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect - github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.15 // indirect - github.com/mdlayher/netlink v1.4.2 // indirect - github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb // indirect + github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect + github.com/mdlayher/socket v0.5.0 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect @@ -128,16 +125,12 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/safchain/ethtool v0.0.0-20200218184317-f459e2d13664 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect - github.com/sirupsen/logrus v1.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/mod v0.37.0 // indirect golang.org/x/text v0.39.0 // indirect - golang.org/x/tools v0.47.0 // indirect - golang.org/x/tools/go/expect v0.1.1-deprecated // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - honnef.co/go/tools v0.2.2 // indirect ) diff --git a/go.sum b/go.sum index 671ba99a9..fad0dffa8 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -33,13 +32,10 @@ github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= -github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= -github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/cilium/ebpf v0.18.0 h1:OsSwqS4y+gQHxaKgg2U/+Fev834kdnsQbtzRnbVC6Gs= +github.com/cilium/ebpf v0.18.0/go.mod h1:vmsAT73y4lW2b4peE+qcOqw6MxvWQdC+LiU5gd/xyo4= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= @@ -51,11 +47,8 @@ github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cvaroqui/ini v1.66.7-0.20220627091046-b218d4fc5c30 h1:XUmTAT3jTtGQ7OX+LKFtZ0i/OcrqGG4aCNmNpQoY8hI= github.com/cvaroqui/ini v1.66.7-0.20220627091046-b218d4fc5c30/go.mod h1:jKhAZrXLB1Q4DwvnR7DC+EfudacHZZ041HBAQtEbNvY= @@ -75,8 +68,8 @@ github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yez github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/eiannone/keyboard v0.0.0-20200508000154-caf4b762e807 h1:jdjd5e68T4R/j4PWxfZqcKY8KtT9oo8IPNVuV4bSXDQ= github.com/eiannone/keyboard v0.0.0-20200508000154-caf4b762e807/go.mod h1:Xoiu5VdKMvbRgHuY7+z64lhu/7lvax/22nzASF6GrO8= @@ -85,9 +78,6 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -116,13 +106,15 @@ github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3U github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -151,20 +143,15 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/nftables v0.0.0-20220129182606-a46119e5928d h1:toAfvkSxI8myTLtOig3ASBHhic8iGVCLAyWsNAczN1g= -github.com/google/nftables v0.0.0-20220129182606-a46119e5928d/go.mod h1:jQsJtZ/NHsAH3Y56xYIctcqNgAymBIJ59dXX/2RsmF4= +github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= +github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -195,18 +182,8 @@ github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6t github.com/jaypipes/pcidb v0.6.0 h1:VIM7GKVaW4qba30cvB67xSCgJPTzkG8Kzw/cbs5PHWU= github.com/jaypipes/pcidb v0.6.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA= -github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= -github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= -github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= -github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw= -github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs= -github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA= -github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U= -github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo= -github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ= -github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= @@ -217,11 +194,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d h1:MFX8DxRnKMY/2M3H61iSsVbo/n3h0MWGmWNN1UViOU0= -github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d/go.mod h1:QHb4k4cr1fQikUahfcRVPcEXiUgFsdIstGqlurL0XL4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -254,27 +228,10 @@ github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= -github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60 h1:tHdB+hQRHU10CfcK0furo6rSNgZ38JT8uPh70c/pFD8= -github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE= -github.com/mdlayher/genetlink v1.0.0 h1:OoHN1OdyEIkScEmRgxLEe2M9U8ClMytqA5niynLtfj0= -github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= -github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= -github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= -github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= -github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= -github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8= -github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU= -github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU= -github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys= -github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8= -github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q= -github.com/mdlayher/netlink v1.4.2 h1:3sbnJWe/LETovA7yRZIX3f9McVOWV3OySH6iIBxiFfI= -github.com/mdlayher/netlink v1.4.2/go.mod h1:13VaingaArGUTUxFLf/iEovKxXji32JAtF858jZYEug= -github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc= -github.com/mdlayher/socket v0.0.0-20211007213009-516dcbdf0267/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g= -github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb h1:2dC7L10LmTqlyMVzFJ00qM25lqESg9Z4u3GuEXN5iHY= -github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o= +github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= +github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -354,12 +311,11 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/safchain/ethtool v0.0.0-20200218184317-f459e2d13664 h1:gvolwzuDhul9qK6/oHqxCHD5TEYfsWNBGidOeG6kvpk= @@ -371,11 +327,9 @@ github.com/shaj13/go-guardian/v2 v2.11.6/go.mod h1:rSe5VLuWu9EyUT68Xi6qxb/DJc+aj github.com/shaj13/libcache v1.0.0/go.mod h1:YCq92Zosqj4erhlLdm2Mu1cX2FDAxjfFOxTphzN7S9U= github.com/shaj13/libcache v1.0.5 h1:oYfQ+TcixPUQp64/DZprH77vMnZ8h6CDtNXuvPmUYdk= github.com/shaj13/libcache v1.0.5/go.mod h1:YCq92Zosqj4erhlLdm2Mu1cX2FDAxjfFOxTphzN7S9U= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= -github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= @@ -403,26 +357,22 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/talos-systems/go-smbios v0.1.1 h1:Au6obB/Pp0i0JHhvPlzONk5aoNseosO2BUsmvWWi7y8= github.com/talos-systems/go-smbios v0.1.1/go.mod h1:vk76naUSZaWE8Z95wbDn51FgH0goECM4oK3KY2hYSMU= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 h1:8mhqcHPqTMhSPoslhGYihEgSfc77+7La1P6kiB6+9So= -github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zcalusic/sysinfo v0.0.0-20210831153053-2c6e1d254246 h1:IPCi0C6XVSrBw6N6awpC+zl29kSJ7z5X+SFvw89wOcQ= github.com/zcalusic/sysinfo v0.0.0-20210831153053-2c6e1d254246/go.mod h1:WGLNaWsjKQ2gXmAHh+MQztgu3FLFAnOFJjFzhpgShCY= @@ -437,43 +387,23 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= -golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= +golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= @@ -484,7 +414,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= @@ -492,49 +421,27 @@ golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= @@ -549,7 +456,6 @@ golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -564,15 +470,8 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= -golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= -golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -616,9 +515,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= -honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk= -honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= From 6ac349b8cd8736549808050bb5a98afcf12f8c31 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 08:04:31 +0200 Subject: [PATCH 02/54] Fix undue ZoneRecordUpdated,ZoneRecordDeleted events - Remove the buggy getExistingRecords function that only indexed by record name - Use Record directly as map keys (since all fields are comparable) - Properly track existing and new records to detect actual changes - Publish ZoneRecordUpdated only when records are new or changed - Publish ZoneRecordDeleted only when records disappear --- daemon/dns/main_cmd.go | 55 +++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/daemon/dns/main_cmd.go b/daemon/dns/main_cmd.go index e6d537bae..cd013b88a 100644 --- a/daemon/dns/main_cmd.go +++ b/daemon/dns/main_cmd.go @@ -104,28 +104,27 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { name := naming.NewFQDN(c.Path, t.clusterConfig.Name).String() + "." nameOnNode := fmt.Sprintf("%s.%s.%s.%s.node.%s.", c.Path.Name, c.Path.Namespace, c.Path.Kind, c.Node, t.clusterConfig.Name) records := make(Zone, 0) - updatedRecords := make(map[string]any) - existingRecords := t.getExistingRecords(key) + existingRecords := t.state[key] + + // Build a map of existing records by their unique identifier + existingRecordsMap := make(map[Record]Record) + for _, record := range existingRecords { + existingRecordsMap[record] = record + } + + // Track which existing records are still present + newRecordsMap := make(map[Record]struct{}) + stage := func(record Record) { records = append(records, record) - existingRecord, ok := existingRecords[record.Name] - var change bool - switch { - case !ok: - change = true - case existingRecord.Content != record.Content: - change = true - case existingRecord.Type != record.Type: - change = true - case existingRecord.DomainID != record.DomainID: - change = true - case existingRecord.TTL != record.TTL: - change = true - } - if change { + newRecordsMap[record] = struct{}{} + + // Check if this exact record already exists + // If not, it means the record is new or has changed (any field difference creates a different key) + if _, ok := existingRecordsMap[record]; !ok { t.pubUpdated(record, c.Path, c.Node) - updatedRecords[record.Name] = nil } + // If the record with the same key exists, it's identical, so no need to publish } stageSRV := func(s string) error { expose, err := ParseExpose(s) @@ -273,11 +272,13 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { stageSRVs(rid, rstat) } - for key, record := range existingRecords { - if _, ok := updatedRecords[key]; !ok { - t.pubDeleted(record, c.Path, c.Node) + // Delete records that no longer exist + for _, existingRecord := range existingRecords { + if _, ok := newRecordsMap[existingRecord]; !ok { + t.pubDeleted(existingRecord, c.Path, c.Node) } } + if len(records) > 0 { t.state[key] = records } else { @@ -341,18 +342,6 @@ func (t *Manager) zone() Zone { return zone } -func (t *Manager) getExistingRecords(key stateKey) map[string]Record { - m := make(map[string]Record) - records, ok := t.state[key] - if !ok { - return m - } - for _, record := range records { - m[record.Name] = record - } - return m -} - func uitoa(val uint) string { if val == 0 { return "0" From 507aa41307756d772206167f28a7795ef86ac683 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 08:48:50 +0200 Subject: [PATCH 03/54] Use a map[recordKey] in the dns to store a instance ips state Instead of a []Record, so we naturally avoid duplicated and we can detect already existing records fast. --- daemon/dns/main.go | 6 ++--- daemon/dns/main_cmd.go | 54 +++++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/daemon/dns/main.go b/daemon/dns/main.go index 900ff9010..77cdca41c 100644 --- a/daemon/dns/main.go +++ b/daemon/dns/main.go @@ -42,9 +42,9 @@ type ( drainDuration time.Duration // state is a map indexed by object path where the key is a zone fragment regrouping all records created for this object. - // Using this map layout permits fast records drop on InstanceStatusDeleted. + // Using this map layout permits fast records drop on InstanceStatusDeleted and safeguards against duplicate records. // The zone data is obtained by merging all map values. - state map[stateKey]Zone + state map[stateKey]map[recordKey]Record // score stores the node.Stats.Score values, to use as weight in SRV records score map[string]int @@ -101,7 +101,7 @@ func NewManager(d time.Duration, subQS pubsub.QueueSizer) *Manager { return &Manager{ cmdC: make(chan any), drainDuration: d, - state: make(map[stateKey]Zone), + state: make(map[stateKey]map[recordKey]Record), score: make(map[string]int), subQS: subQS, diff --git a/daemon/dns/main_cmd.go b/daemon/dns/main_cmd.go index cd013b88a..53854b3fb 100644 --- a/daemon/dns/main_cmd.go +++ b/daemon/dns/main_cmd.go @@ -38,6 +38,13 @@ func (t *Manager) stateKey(p naming.Path, node string) stateKey { } } +// recordKey uniquely identifies a DNS record (excluding TTL and DomainID which are metadata) +type recordKey struct { + Name string + Type string + Content string +} + func (t *Manager) onNodeStatsUpdated(c *msgbus.NodeStatsUpdated) { t.score[c.Node] = c.Value.Score } @@ -91,8 +98,8 @@ func (t *Manager) pubUpdated(record Record, p naming.Path, node string) { func (t *Manager) onInstanceStatusDeleted(c *msgbus.InstanceStatusDeleted) { key := t.stateKey(c.Path, c.Node) - if records, ok := t.state[key]; ok { - for _, record := range records { + if recordMap, ok := t.state[key]; ok { + for _, record := range recordMap { t.pubDeleted(record, c.Path, c.Node) } delete(t.state, key) @@ -103,28 +110,22 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { key := t.stateKey(c.Path, c.Node) name := naming.NewFQDN(c.Path, t.clusterConfig.Name).String() + "." nameOnNode := fmt.Sprintf("%s.%s.%s.%s.node.%s.", c.Path.Name, c.Path.Namespace, c.Path.Kind, c.Node, t.clusterConfig.Name) - records := make(Zone, 0) - existingRecords := t.state[key] - - // Build a map of existing records by their unique identifier - existingRecordsMap := make(map[Record]Record) - for _, record := range existingRecords { - existingRecordsMap[record] = record - } - - // Track which existing records are still present - newRecordsMap := make(map[Record]struct{}) + newRecordsMap := make(map[recordKey]Record) + existingRecordsMap := t.state[key] stage := func(record Record) { - records = append(records, record) - newRecordsMap[record] = struct{}{} + recordKey := recordKey{record.Name, record.Type, record.Content} - // Check if this exact record already exists - // If not, it means the record is new or has changed (any field difference creates a different key) - if _, ok := existingRecordsMap[record]; !ok { + // Check if this record already exists (by identity, not by TTL/DomainID) + if existingRecord, ok := existingRecordsMap[recordKey]; !ok { + // New record, publish update + t.pubUpdated(record, c.Path, c.Node) + } else if existingRecord != record { + // Record exists but has changed (TTL or DomainID difference) t.pubUpdated(record, c.Path, c.Node) } - // If the record with the same key exists, it's identical, so no need to publish + // Store in new records map (preserves the full Record with current TTL/DomainID) + newRecordsMap[recordKey] = record } stageSRV := func(s string) error { expose, err := ParseExpose(s) @@ -273,14 +274,15 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { } // Delete records that no longer exist - for _, existingRecord := range existingRecords { - if _, ok := newRecordsMap[existingRecord]; !ok { + for recordKey, existingRecord := range existingRecordsMap { + if _, ok := newRecordsMap[recordKey]; !ok { t.pubDeleted(existingRecord, c.Path, c.Node) } } - if len(records) > 0 { - t.state[key] = records + // Update state: if we have new records, store them; otherwise clean up + if len(newRecordsMap) > 0 { + t.state[key] = newRecordsMap } else { delete(t.state, key) } @@ -336,8 +338,10 @@ func (t *Manager) zone() Zone { }, ) } - for _, records := range t.state { - zone = append(zone, records...) + for _, recordMap := range t.state { + for _, record := range recordMap { + zone = append(zone, record) + } } return zone } From d1325e01578b4d6ed982d7e769be0c1982f1bc45 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 09:26:49 +0200 Subject: [PATCH 04/54] Remove the "alive" dns uds socket trace log This is repeating every second for no reason: we have traces on socket state transitions already. --- daemon/dns/uds.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/dns/uds.go b/daemon/dns/uds.go index e18ced95e..3a3a64ab5 100644 --- a/daemon/dns/uds.go +++ b/daemon/dns/uds.go @@ -330,7 +330,8 @@ func (t *Manager) startUDSListener() error { message = buffer[:n] if os.IsTimeout(err) { - t.log.Tracef("%d: alive", id) + // Only add this trace if needed, as it is logging every second + //t.log.Tracef("%d: alive", id) continue } else if errors.Is(err, io.EOF) { t.log.Tracef("%d: close connection (%s), served %d requests", id, err, reqCount) From d98b2a3706f0afacee64ac7364c88c1f400c008b Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 09:47:53 +0200 Subject: [PATCH 05/54] Truncate the dns uds response trace log to 1024 bytes Protection against log flood and buffer DoS. --- daemon/dns/uds.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/daemon/dns/uds.go b/daemon/dns/uds.go index 3a3a64ab5..d33d4ff30 100644 --- a/daemon/dns/uds.go +++ b/daemon/dns/uds.go @@ -268,7 +268,11 @@ func (t *Manager) startUDSListener() error { sendBytes := func(id uint64, conn net.Conn, b []byte) error { b = append(b, []byte("\n")...) - t.log.Tracef("%d: >>> %s", id, string(b)) + content := string(b) + if len(content) > 1024 { + content = content[:1024] + "..." + } + t.log.Tracef("%d: >>> %s", id, content) if err := conn.SetWriteDeadline(time.Now().Add(time.Second)); err != nil { t.log.Warnf("%d: can't set response write deadline: %s", id, err) } From bfa8cc4ab3b73b8df5f6355bf42e2cbc1449e16d Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 09:53:02 +0200 Subject: [PATCH 06/54] More dns fixes - Added validation to skip empty and invalid IP addresses - Deduplicate Records in Zone --- daemon/dns/main_cmd.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/daemon/dns/main_cmd.go b/daemon/dns/main_cmd.go index 53854b3fb..b7b749cce 100644 --- a/daemon/dns/main_cmd.go +++ b/daemon/dns/main_cmd.go @@ -169,10 +169,13 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { continue } ipAddr, ok := i.(string) - if !ok { + if !ok || ipAddr == "" { continue } ip := net.ParseIP(ipAddr) + if ip == nil { + continue + } isIPV4 := ip.To4() != nil var aType, ptrType string if isIPV4 { @@ -338,9 +341,14 @@ func (t *Manager) zone() Zone { }, ) } + seen := make(map[recordKey]bool) for _, recordMap := range t.state { for _, record := range recordMap { - zone = append(zone, record) + key := recordKey{record.Name, record.Type, record.Content} + if !seen[key] { + zone = append(zone, record) + seen[key] = true + } } } return zone From 8d57bc5a8f319848b87d0c436354fedb56925218 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 10:41:04 +0200 Subject: [PATCH 07/54] Use a dns name cache to serve lookups faster The index is rebuild on: - InstanceStatusUpdated (Instance records added/updated) - InstanceStatusDeleted (Instance records removed) - ClusterConfigUpdated (SOA/NS records depend on clusterConfig.DNS) --- daemon/dns/main.go | 4 +++ daemon/dns/main_cmd.go | 75 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/daemon/dns/main.go b/daemon/dns/main.go index 77cdca41c..4ce95d059 100644 --- a/daemon/dns/main.go +++ b/daemon/dns/main.go @@ -46,6 +46,9 @@ type ( // The zone data is obtained by merging all map values. state map[stateKey]map[recordKey]Record + // nameIndex maps record names to their records for O(1) lookups + nameIndex map[string][]Record + // score stores the node.Stats.Score values, to use as weight in SRV records score map[string]int @@ -102,6 +105,7 @@ func NewManager(d time.Duration, subQS pubsub.QueueSizer) *Manager { cmdC: make(chan any), drainDuration: d, state: make(map[stateKey]map[recordKey]Record), + nameIndex: make(map[string][]Record), score: make(map[string]int), subQS: subQS, diff --git a/daemon/dns/main_cmd.go b/daemon/dns/main_cmd.go index b7b749cce..a7e7245da 100644 --- a/daemon/dns/main_cmd.go +++ b/daemon/dns/main_cmd.go @@ -72,6 +72,8 @@ func (t *Manager) onClusterConfigUpdated(c *msgbus.ClusterConfigUpdated) { if change { t.publishSubsystemDnsUpdated() } + // Rebuild nameIndex since SOA/NS records depend on clusterConfig.DNS + t.rebuildNameIndex() } func (t *Manager) pubDeleted(record Record, p naming.Path, node string) { @@ -104,6 +106,8 @@ func (t *Manager) onInstanceStatusDeleted(c *msgbus.InstanceStatusDeleted) { } delete(t.state, key) } + // Rebuild nameIndex for O(1) lookups + t.rebuildNameIndex() } func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { @@ -289,18 +293,30 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { } else { delete(t.state, key) } + // Rebuild nameIndex for O(1) lookups + t.rebuildNameIndex() } func (t *Manager) onCmdGet(c cmdGet) { - zone := make(Zone, 0) - for _, record := range t.zone() { - if record.Name != c.Name { - continue - } + // Use nameIndex for O(1) lookup + records, ok := t.nameIndex[c.Name] + if !ok { + c.errC <- nil + c.resp <- Zone{} + return + } + // Pre-size slice with estimated capacity (Fix 3) + zone := make(Zone, 0, len(records)) + seen := make(map[recordKey]bool) + for _, record := range records { if (c.Type != "ANY") && (record.Type != c.Type) { continue } - zone = append(zone, record) + key := recordKey{record.Name, record.Type, record.Content} + if !seen[key] { + zone = append(zone, record) + seen[key] = true + } } c.errC <- nil c.resp <- zone @@ -354,6 +370,53 @@ func (t *Manager) zone() Zone { return zone } +// rebuildNameIndex rebuilds the name-to-records index including both +// state records and cluster-level SOA/NS records +func (t *Manager) rebuildNameIndex() { + t.nameIndex = make(map[string][]Record) + + // Add cluster-level SOA and NS records + zoneName := t.clusterConfig.Name + "." + for i, dns := range t.clusterConfig.DNS { + nsName := fmt.Sprintf("ns%d.%s", i+1, zoneName) + soaContent := fmt.Sprintf("dns.%s %s %d %d %d %d %d", zoneName, contact, serial, refresh, retry, expire, minimum) + + // SOA record + t.nameIndex[zoneName] = append(t.nameIndex[zoneName], Record{ + Name: zoneName, + DomainID: -1, + Type: "SOA", + TTL: 60, + Content: soaContent, + }) + + // A record for nameserver + t.nameIndex[nsName] = append(t.nameIndex[nsName], Record{ + Name: nsName, + DomainID: -1, + Type: "A", + TTL: 60, + Content: dns, + }) + + // NS record + t.nameIndex[zoneName] = append(t.nameIndex[zoneName], Record{ + Name: zoneName, + DomainID: -1, + Type: "NS", + TTL: 3600, + Content: nsName, + }) + } + + // Add instance records from state + for _, recordMap := range t.state { + for _, record := range recordMap { + t.nameIndex[record.Name] = append(t.nameIndex[record.Name], record) + } + } +} + func uitoa(val uint) string { if val == 0 { return "0" From b392943c41f9dbe53950a52b16fcf4c5f17ab121 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 12:06:09 +0200 Subject: [PATCH 08/54] Don't parse /proc/mountinfo on every poll timeout Look at the event counter in the Poll() result to decide if reparsing is needed. Add a prom counter for total number of /proc/mountinfo parsing. $ curl -s -k https://localhost:1215/metrics| egrep ^opensvc_mntmon opensvc_mntmon_parse_mountinfo_total 4 --- daemon/mntmon/main.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/daemon/mntmon/main.go b/daemon/mntmon/main.go index ce888e071..1a1bfa35e 100644 --- a/daemon/mntmon/main.go +++ b/daemon/mntmon/main.go @@ -16,6 +16,8 @@ import ( "github.com/opensvc/om3/v3/util/hostname" "github.com/opensvc/om3/v3/util/plog" "github.com/opensvc/om3/v3/util/pubsub" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "golang.org/x/sys/unix" ) @@ -26,6 +28,7 @@ type mountEntry struct { } func parseMountinfo(path string) (map[string]mountEntry, error) { + mntmonParseMountinfoTotal.Inc() f, err := os.Open(path) if err != nil { return nil, err @@ -89,6 +92,17 @@ type ( } ) +var ( + mntmonParseMountinfoTotal = promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: "opensvc", + Subsystem: "mntmon", + Name: "parse_mountinfo_total", + Help: "Total number of /proc/self/mountinfo parsing calls", + }, + ) +) + // NewManager creates a new mount monitor manager func NewManager(drainDuration time.Duration, subQS pubsub.QueueSizer) *Manager { localhost := hostname.Hostname() @@ -197,7 +211,7 @@ func (t *Manager) watchMounts() { } // Poll with finite timeout - _, err := unix.Poll(pfd, int(pollTimeout.Milliseconds())) + n, err := unix.Poll(pfd, int(pollTimeout.Milliseconds())) if err != nil { if err == unix.EINTR { continue @@ -214,6 +228,11 @@ func (t *Manager) watchMounts() { default: } + if n == 0 { + // no event, poll just timed out + continue + } + cur, err := parseMountinfo(mountinfoPath) if err != nil { t.log.Errorf("parse mountinfo: %s", err) From a8295f87c65c1885a667688462a9ff4f2c1f3004 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 15:45:02 +0200 Subject: [PATCH 09/54] Implement tcp conn pooling in hb.ucast The net.(*Dialer).Dial functions (which were consuming ~36% of CPU in the network category) have been reduced to ~5.88%, representing the connection management overhead for new connections (e.g., when nodes restart). --- daemon/hb/hbucast/hbrx.go | 81 +++++++++++++++++- daemon/hb/hbucast/hbtx.go | 168 +++++++++++++++++++++++++------------- 2 files changed, 190 insertions(+), 59 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index ac4d25fda..f256851d5 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -212,7 +212,7 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { wg.Add(1) go func() { defer wg.Done() - t.handle(clearConn) + t.handleLoop(clearConn) }() } wg.Wait() @@ -224,6 +224,8 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { } func (t *rx) handle(conn encryptconn.ConnNoder) { + // For backward compatibility with old senders that create new connections per message, + // we read one message and close the connection defer func() { if err := conn.Close(); err != nil { t.log.Warnf("unexpected error while closing connection from %s: %s", conn.RemoteAddr(), err) @@ -231,11 +233,15 @@ func (t *rx) handle(conn encryptconn.ConnNoder) { }() data := msgPool.Get().([]byte) defer func() { msgPool.Put(data) }() + + t.log.Tracef("handle: reading single message from %s", conn.RemoteAddr()) i, nodename, err := conn.ReadWithNode(data) if err != nil { - t.log.Warnf("read failed from %s: %s", conn.RemoteAddr(), err) + t.log.Warnf("handle: read failed from %s: %s", conn.RemoteAddr(), err) return } + t.log.Tracef("handle: received %d bytes from node %s", i, nodename) + if i >= (msgMaxSize - 10000) { t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, conn.RemoteAddr(), i) } @@ -244,6 +250,8 @@ func (t *rx) handle(conn encryptconn.ConnNoder) { t.log.Warnf("unmarshal message failed from node %s:%s: %s", nodename, conn.RemoteAddr(), err) return } + t.log.Tracef("handle: successfully decoded message from node %s (kind=%s)", nodename, msg.Kind) + cmdPeerSuccess := hbctrl.CmdSetPeerSuccess{ Nodename: msg.Nodename, HbID: t.id, @@ -260,6 +268,75 @@ func (t *rx) handle(conn encryptconn.ConnNoder) { } } +func (t *rx) handleLoop(conn encryptconn.ConnNoder) { + // Read messages in a loop to support connection reuse from pool on sender side + // When sender uses connection pooling, multiple messages are sent on the same connection. + // We don't set a read deadline here to avoid timing out on idle connections between heartbeats. + // The connection will be closed by the sender when it's done. + defer func() { + t.log.Tracef("handleLoop: connection closed from %s", conn.RemoteAddr()) + if err := conn.Close(); err != nil { + t.log.Warnf("unexpected error while closing connection from %s: %s", conn.RemoteAddr(), err) + } + }() + + t.log.Tracef("handleLoop: starting to read messages from %s", conn.RemoteAddr()) + + data := msgPool.Get().([]byte) + defer func() { msgPool.Put(data) }() + + msgCount := 0 + for { + // Reset buffer for each message + buffer := data[:cap(data)] + + // No deadline - connection is kept open by sender with persistent connections + // Read will block until data arrives or connection is closed + i, nodename, err := conn.ReadWithNode(buffer) + if err != nil { + if err.Error() != "EOF" { + t.log.Tracef("handleLoop: read failed from %s after %d messages: %s", conn.RemoteAddr(), msgCount, err) + } else { + t.log.Tracef("handleLoop: EOF from %s after %d messages", conn.RemoteAddr(), msgCount) + } + return + } + msgCount++ + + t.log.Tracef("handleLoop: received %d bytes from node %s (msg #%d)", i, nodename, msgCount) + + if i >= (msgMaxSize - 10000) { + t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, conn.RemoteAddr(), i) + } + msg := hbtype.Msg{} + if err := json.Unmarshal(buffer[:i], &msg); err != nil { + t.log.Warnf("handleLoop: unmarshal message failed from node %s:%s: %s", nodename, conn.RemoteAddr(), err) + return + } + t.log.Tracef("handleLoop: successfully decoded message from node %s (kind=%s)", nodename, msg.Kind) + + cmdPeerSuccess := hbctrl.CmdSetPeerSuccess{ + Nodename: msg.Nodename, + HbID: t.id, + Success: true, + } + select { + case <-t.ctx.Done(): + t.log.Tracef("handleLoop: context done, stopping after %d messages", msgCount) + return + case t.cmdC <- cmdPeerSuccess: + t.log.Tracef("handleLoop: sent peer success for node %s", msg.Nodename) + } + select { + case <-t.ctx.Done(): + t.log.Tracef("handleLoop: context done while sending msg, stopping after %d messages", msgCount) + return + case t.msgC <- &msg: + t.log.Tracef("handleLoop: sent message to msgC (node=%s, kind=%s)", msg.Nodename, msg.Kind) + } + } +} + func newRx(ctx context.Context, name string, nodes map[string]string, addr, port, intf string, timeout time.Duration) *rx { id := name + ".rx" return &rx{ diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 2b48b0e3b..86762313c 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -30,11 +30,14 @@ type ( localIP net.IP lastNodeErr sync.Map - name string - log *plog.Logger - cmdC chan<- interface{} - msgC chan<- *hbtype.Msg - cancel func() + name string + log *plog.Logger + cmdC chan<- interface{} + msgC chan<- *hbtype.Msg + cancel func() + // Per-peer connections that are kept open + peerConns map[string]net.Conn + peerMutex sync.Mutex } ) @@ -53,6 +56,13 @@ func (t *tx) Stop() error { Nodename: node, } } + // Close all peer connections + t.peerMutex.Lock() + for _, conn := range t.peerConns { + conn.Close() + } + t.peerConns = make(map[string]net.Conn) + t.peerMutex.Unlock() t.Wait() t.log.Tracef("wait done") return nil @@ -85,6 +95,7 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { t.ctx = ctx t.cancel = cancel t.cmdC = cmdC + t.peerConns = make(map[string]net.Conn) t.Add(1) hbaudit.EnableAudit(ctx, t.id, t.log, "hb", strings.Replace(t.id, "hb#", "hb:", 1)) @@ -175,70 +186,50 @@ func (t *tx) defaultLocalIP() (net.IP, error) { } func (t *tx) send(node, addr string, b []byte) { + t.log.Tracef("send: starting for node %s, addr %s, data len %d", node, addr, len(b)) localAddr := net.TCPAddr{ IP: t.localIP, Port: 0, } - dialer := net.Dialer{ - Timeout: t.timeout, - LocalAddr: &localAddr, - } - send := func() error { - conn, err := dialer.Dial("tcp", addr) - if err != nil { - return err - } - defer func() { - _ = conn.Close() - }() - if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { - return err - } - if n, err := conn.Write(b); err != nil { - return err - } else if n != len(b) { - return err - } - return nil - } - clearDedupLog := func() { - if lastErr, ok := t.lastNodeErr.Load(node); !ok { - return - } else { - t.log.Infof("end a send error period for node %s: %s", node, lastErr) - t.lastNodeErr.Delete(node) - } + // Get or create persistent connection for this peer + rawConn, err := t.getPeerConn(node, addr, localAddr) + if err != nil { + t.handleSendError(node, err) + return } - // setDedupLog manages the logging of consecutive errors for a specific node. - // It is designed to prevent log spam by only reporting the start and end of an - // error "period" and not every single occurrence of the same error. - // The function uses a sync.Map (t.lastNodeErr) to safely store the last - // recorded error string for each node, which is essential for concurrent access. - setDedupLog := func(err error) { - newErr := err.Error() - if lastErr, ok := t.lastNodeErr.Load(node); ok { - if lastErr == newErr { - return - } else if lastErr != "" { - t.log.Infof("end a send error period for node %s: %s", node, lastErr) - } - } - if newErr != "" { - t.log.Warnf("begin a send error period for node %s: %s", node, newErr) - t.lastNodeErr.Store(node, newErr) - } else { - t.lastNodeErr.Delete(node) - } + // Set deadline on the raw connection + if err := rawConn.SetDeadline(time.Now().Add(t.timeout)); err != nil { + t.handleSendError(node, err) + return } - if err := send(); err != nil { - setDedupLog(err) + // Create encryptconn wrapper to add null terminator + // The data in b is already encrypted by the main hb code, but we need to add + // the null terminator that encryptconn.ReadWithNode expects + // However, b is already the final encrypted message, so we need to write it + // with the null terminator + // For now, just write b + null byte directly + dataWithTerminator := append(b, 0x00) + + t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(b), 1, addr) + if n, err := rawConn.Write(dataWithTerminator); err != nil { + t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) + t.handleSendError(node, err) + // Remove the connection so we can reconnect next time + t.removePeerConn(node) + return + } else if n != len(dataWithTerminator) { + t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(dataWithTerminator)) + t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(dataWithTerminator))) + // Remove the connection so we can reconnect next time + t.removePeerConn(node) return } - clearDedupLog() + t.log.Tracef("successfully sent %d bytes to %s", len(b), addr) + t.clearDedupLog(node) t.cmdC <- hbctrl.CmdSetPeerSuccess{ Nodename: node, @@ -247,6 +238,69 @@ func (t *tx) send(node, addr string, b []byte) { } } +// getPeerConn returns the persistent connection for a peer, creating it if necessary +func (t *tx) getPeerConn(node, addr string, localAddr net.TCPAddr) (net.Conn, error) { + t.peerMutex.Lock() + defer t.peerMutex.Unlock() + + if conn, exists := t.peerConns[node]; exists { + return conn, nil + } + + // Create new connection + dialer := net.Dialer{ + Timeout: t.timeout, + LocalAddr: &localAddr, + } + conn, err := dialer.Dial("tcp", addr) + if err != nil { + return nil, err + } + + t.peerConns[node] = conn + t.log.Tracef("created new persistent connection to %s (%s)", node, addr) + return conn, nil +} + +// removePeerConn removes a peer connection from the map +func (t *tx) removePeerConn(node string) { + t.peerMutex.Lock() + defer t.peerMutex.Unlock() + if conn, exists := t.peerConns[node]; exists { + conn.Close() + delete(t.peerConns, node) + t.log.Tracef("removed connection to %s", node) + } +} + +// handleSendError handles send errors with deduplication logging +func (t *tx) handleSendError(node string, err error) { + newErr := err.Error() + if lastErr, ok := t.lastNodeErr.Load(node); ok { + if lastErr == newErr { + return + } else if lastErr != "" { + t.log.Infof("end a send error period for node %s: %s", node, lastErr) + } + } + if newErr != "" { + t.log.Warnf("begin a send error period for node %s: %s", node, newErr) + t.lastNodeErr.Store(node, newErr) + } else { + t.lastNodeErr.Delete(node) + } +} + +// clearDedupLog clears the deduplication log for a node +func (t *tx) clearDedupLog(node string) { + if lastErr, ok := t.lastNodeErr.Load(node); !ok { + return + } else { + t.log.Infof("end a send error period for node %s: %s", node, lastErr) + t.lastNodeErr.Delete(node) + } +} + func newTx(ctx context.Context, name string, nodes map[string]string, addr, port, intf string, timeout, interval time.Duration) *tx { id := name + ".tx" return &tx{ From ce5d666ffc582e143c6ebd159cdf7528763932d5 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 16:36:03 +0200 Subject: [PATCH 10/54] make dns ignore the ipaddr when the resource is not up So we don't include in the zone records for scoped ip.host addr when the instance resource is not up. --- daemon/dns/main_cmd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/daemon/dns/main_cmd.go b/daemon/dns/main_cmd.go index a7e7245da..aefc81db2 100644 --- a/daemon/dns/main_cmd.go +++ b/daemon/dns/main_cmd.go @@ -8,6 +8,7 @@ import ( "github.com/opensvc/om3/v3/core/naming" "github.com/opensvc/om3/v3/core/resource" "github.com/opensvc/om3/v3/core/resourceid" + "github.com/opensvc/om3/v3/core/status" "github.com/opensvc/om3/v3/daemon/msgbus" "github.com/opensvc/om3/v3/util/pubsub" ) @@ -168,6 +169,9 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { } } for rid, rstat := range c.Value.Resources { + if !rstat.Status.Is(status.Up) { + continue + } i, ok := rstat.Info[ipAddrInfoKey] if !ok { continue From 9575c5c71c04a3d89949e1efa683c3f2054169bc Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 17:04:04 +0200 Subject: [PATCH 11/54] Better dns uds trace logs - Avoid unnecessary string(b) - Avoid adding line feeds in traces --- daemon/dns/uds.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/daemon/dns/uds.go b/daemon/dns/uds.go index d33d4ff30..c9b7e18e7 100644 --- a/daemon/dns/uds.go +++ b/daemon/dns/uds.go @@ -1,6 +1,7 @@ package dns import ( + "bytes" "encoding/json" "errors" "fmt" @@ -267,12 +268,12 @@ func (t *Manager) startUDSListener() error { } sendBytes := func(id uint64, conn net.Conn, b []byte) error { - b = append(b, []byte("\n")...) - content := string(b) - if len(content) > 1024 { - content = content[:1024] + "..." + if len(b) > 1024 { + t.log.Tracef("%d: >>> %s...", id, b[:1024]) + } else { + t.log.Tracef("%d: >>> %s", id, b) } - t.log.Tracef("%d: >>> %s", id, content) + b = append(b, []byte("\n")...) if err := conn.SetWriteDeadline(time.Now().Add(time.Second)); err != nil { t.log.Warnf("%d: can't set response write deadline: %s", id, err) } @@ -296,7 +297,7 @@ func (t *Manager) startUDSListener() error { Result: false, } b, _ := json.Marshal(response) - t.log.Tracef("%d: >>> %s", id, string(b)) + t.log.Tracef("%d: >>> %s", id, b) return sendBytes(id, conn, b) } @@ -351,7 +352,7 @@ func (t *Manager) startUDSListener() error { } reqCount++ - t.log.Tracef("%d: <<< %s", id, string(message)) + t.log.Tracef("%d: <<< %s", id, bytes.TrimRight(message, "\r\n")) if err := json.Unmarshal(message, &req); err != nil { t.log.Errorf("%d: close connection (%s), served %d requests", id, err, reqCount) From ac8ae587b4b454a1d7b85464fd2be92d1cdeeac5 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 17:49:48 +0200 Subject: [PATCH 12/54] Remove dead func from hb.ucast --- daemon/hb/hbucast/hbrx.go | 45 --------------------------------------- 1 file changed, 45 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index f256851d5..456da5167 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -223,51 +223,6 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { return nil } -func (t *rx) handle(conn encryptconn.ConnNoder) { - // For backward compatibility with old senders that create new connections per message, - // we read one message and close the connection - defer func() { - if err := conn.Close(); err != nil { - t.log.Warnf("unexpected error while closing connection from %s: %s", conn.RemoteAddr(), err) - } - }() - data := msgPool.Get().([]byte) - defer func() { msgPool.Put(data) }() - - t.log.Tracef("handle: reading single message from %s", conn.RemoteAddr()) - i, nodename, err := conn.ReadWithNode(data) - if err != nil { - t.log.Warnf("handle: read failed from %s: %s", conn.RemoteAddr(), err) - return - } - t.log.Tracef("handle: received %d bytes from node %s", i, nodename) - - if i >= (msgMaxSize - 10000) { - t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, conn.RemoteAddr(), i) - } - msg := hbtype.Msg{} - if err := json.Unmarshal(data[:i], &msg); err != nil { - t.log.Warnf("unmarshal message failed from node %s:%s: %s", nodename, conn.RemoteAddr(), err) - return - } - t.log.Tracef("handle: successfully decoded message from node %s (kind=%s)", nodename, msg.Kind) - - cmdPeerSuccess := hbctrl.CmdSetPeerSuccess{ - Nodename: msg.Nodename, - HbID: t.id, - Success: true, - } - select { - case <-t.ctx.Done(): - return - case t.cmdC <- cmdPeerSuccess: - } - select { - case <-t.ctx.Done(): - case t.msgC <- &msg: - } -} - func (t *rx) handleLoop(conn encryptconn.ConnNoder) { // Read messages in a loop to support connection reuse from pool on sender side // When sender uses connection pooling, multiple messages are sent on the same connection. From 7a4e93652ff6eecbe0f873dc9c8c00c8ebe900c3 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 26 Aug 2026 20:01:52 +0200 Subject: [PATCH 13/54] Don't set a deadline on hb.ucast rx sock operations Because, on deadline the socket is closed and it forces tx to close the socket too, making the pool optimization inefficient. --- daemon/hb/hbucast/hbrx.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 456da5167..48acd3ccb 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -40,9 +40,6 @@ type ( ) var ( - // messageTimeout - messageTimeout = 500 * time.Millisecond - msgMaxSize = 10000000 // max kind=full msg size // Create a new sync.Pool to manage the byte buffers. Used to reduce memory usage @@ -204,10 +201,6 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { } continue } - if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil { - t.log.Infof("can't set read deadline for %s: %s", connAddr, err) - continue - } clearConn := encryptconn.New(conn, crypto.Load()) wg.Add(1) go func() { From 3988c20a33e7ac1eb4f16fd552b79011030b94cc Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 09:08:08 +0200 Subject: [PATCH 14/54] Bootstrap the dns zone before serving the first uds request At least the SOA and NS will be available. --- daemon/dns/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/dns/main.go b/daemon/dns/main.go index 4ce95d059..3e746b4a9 100644 --- a/daemon/dns/main.go +++ b/daemon/dns/main.go @@ -128,6 +128,7 @@ func (t *Manager) Start(parent context.Context) error { t.startSubscriptions() t.clusterConfig = *cluster.ConfigData.Get() + t.rebuildNameIndex() if err := t.startUDSListener(); err != nil { return err From 8a62e1065b0e214934260f796e196aff879e3122 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 11:07:19 +0200 Subject: [PATCH 15/54] Fix a hb.ucast locking issue Do not hold a global mutex while dialing. Use per-peer connection locks so one peer's dial timeout cannot block unrelated peers. --- daemon/hb/hbucast/hbtx.go | 47 +++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 86762313c..b41d2dcb7 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -35,9 +35,10 @@ type ( cmdC chan<- interface{} msgC chan<- *hbtype.Msg cancel func() - // Per-peer connections that are kept open - peerConns map[string]net.Conn - peerMutex sync.Mutex + // Per-peer connections that are kept open (sync.Map for concurrent access) + peerConns sync.Map + // Per-peer connection locks to prevent duplicate dial attempts + peerLocks sync.Map } ) @@ -57,12 +58,11 @@ func (t *tx) Stop() error { } } // Close all peer connections - t.peerMutex.Lock() - for _, conn := range t.peerConns { + t.peerConns.Range(func(key, value any) bool { + conn := value.(net.Conn) conn.Close() - } - t.peerConns = make(map[string]net.Conn) - t.peerMutex.Unlock() + return true + }) t.Wait() t.log.Tracef("wait done") return nil @@ -95,7 +95,6 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { t.ctx = ctx t.cancel = cancel t.cmdC = cmdC - t.peerConns = make(map[string]net.Conn) t.Add(1) hbaudit.EnableAudit(ctx, t.id, t.log, "hb", strings.Replace(t.id, "hb#", "hb:", 1)) @@ -240,14 +239,18 @@ func (t *tx) send(node, addr string, b []byte) { // getPeerConn returns the persistent connection for a peer, creating it if necessary func (t *tx) getPeerConn(node, addr string, localAddr net.TCPAddr) (net.Conn, error) { - t.peerMutex.Lock() - defer t.peerMutex.Unlock() + // Get or create per-peer lock using sync.Map + lockI, _ := t.peerLocks.LoadOrStore(node, &sync.Mutex{}) + lock := lockI.(*sync.Mutex) + lock.Lock() + defer lock.Unlock() - if conn, exists := t.peerConns[node]; exists { - return conn, nil + // Check if connection already exists (under per-peer lock) + if connI, exists := t.peerConns.Load(node); exists { + return connI.(net.Conn), nil } - // Create new connection + // Create new connection outside the global lock to avoid blocking other peers dialer := net.Dialer{ Timeout: t.timeout, LocalAddr: &localAddr, @@ -257,18 +260,24 @@ func (t *tx) getPeerConn(node, addr string, localAddr net.TCPAddr) (net.Conn, er return nil, err } - t.peerConns[node] = conn + // Store the connection (still under per-peer lock) + t.peerConns.Store(node, conn) t.log.Tracef("created new persistent connection to %s (%s)", node, addr) return conn, nil } // removePeerConn removes a peer connection from the map func (t *tx) removePeerConn(node string) { - t.peerMutex.Lock() - defer t.peerMutex.Unlock() - if conn, exists := t.peerConns[node]; exists { + // Use per-peer lock to avoid blocking other peers + lockI, _ := t.peerLocks.LoadOrStore(node, &sync.Mutex{}) + lock := lockI.(*sync.Mutex) + lock.Lock() + defer lock.Unlock() + + if connI, exists := t.peerConns.Load(node); exists { + conn := connI.(net.Conn) conn.Close() - delete(t.peerConns, node) + t.peerConns.Delete(node) t.log.Tracef("removed connection to %s", node) } } From dec51a3738208543541d42e9532918762ee724fd Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 11:44:16 +0200 Subject: [PATCH 16/54] Serialize sends to the same node in hb.ucast to prevent goroutine accumulation When a node (e.g. dev2n2) is in a network blackhole, the hb.ucast tx keeps starting goroutines for every send message. Due to the blackhole, the send never finishes and goroutines accumulate, leading to a memory leak. This change implements per-node send serialization: - Each node gets its own buffered channel (size 1) for send requests - A single worker goroutine per node processes sends sequentially - If a send is already in progress, new sends to the same node wait in the queue (or are dropped if the queue is full) - Other nodes are not affected by a blackholed node - Proper cleanup of send queues and worker goroutines on Stop() This ensures that even if a node is unreachable, only one goroutine per node is blocked, preventing unbounded goroutine accumulation. --- daemon/hb/hbucast/hbtx.go | 71 ++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index b41d2dcb7..e2678c981 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -30,18 +30,29 @@ type ( localIP net.IP lastNodeErr sync.Map - name string - log *plog.Logger - cmdC chan<- interface{} - msgC chan<- *hbtype.Msg - cancel func() + name string + log *plog.Logger + cmdC chan<- interface{} + msgC chan<- *hbtype.Msg + cancel func() // Per-peer connections that are kept open (sync.Map for concurrent access) peerConns sync.Map // Per-peer connection locks to prevent duplicate dial attempts peerLocks sync.Map + // Per-peer send queues to serialize sends to the same node + sendQueues sync.Map + // WaitGroup for send worker goroutines + sendWorkers sync.WaitGroup } ) +// sendRequest holds data for a send operation +type sendRequest struct { + node string + addr string + data []byte +} + // ID implements the ID function of Transmitter interface for tx func (t *tx) ID() string { return t.id @@ -63,7 +74,14 @@ func (t *tx) Stop() error { conn.Close() return true }) + // Close all send queues to unblock workers + t.sendQueues.Range(func(key, value any) bool { + q := value.(chan sendRequest) + close(q) + return true + }) t.Wait() + t.sendWorkers.Wait() t.log.Tracef("wait done") return nil } @@ -88,6 +106,35 @@ func (t *tx) streamPeerDesc(addr string) string { } } +// sendToNode queues a send request for a specific node +// It creates a worker goroutine for the node if one doesn't exist +func (t *tx) sendToNode(node, addr string, b []byte) { + // Get or create send queue for this node + queueI, loaded := t.sendQueues.LoadOrStore(node, make(chan sendRequest, 1)) + queue := queueI.(chan sendRequest) + + // If this is the first time we're creating the queue, start a worker + if !loaded { + t.sendWorkers.Add(1) + go func(node, addr string, q chan sendRequest) { + defer t.sendWorkers.Done() + for req := range q { + t.doSend(req.node, req.addr, req.data) + } + }(node, addr, queue) + } + + // Try to send without blocking first (non-blocking send) + select { + case queue <- sendRequest{node: node, addr: addr, data: b}: + // Successfully queued + default: + // Queue is full, drop the message to avoid blocking + // This means a send is already in progress and we don't want to stack up + t.log.Tracef("send queue full for node %s, dropping message", node) + } +} + // Start implements the Start function of Transmitter interface for tx func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { started := make(chan bool) @@ -158,7 +205,7 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { protectedB := make([]byte, len(b)) copy(protectedB, b) for node, addr := range t.nodes { - go t.send(node, addr, protectedB) + t.sendToNode(node, addr, protectedB) } } } @@ -184,7 +231,7 @@ func (t *tx) defaultLocalIP() (net.IP, error) { return addrs[0].IP, nil } -func (t *tx) send(node, addr string, b []byte) { +func (t *tx) doSend(node, addr string, b []byte) { t.log.Tracef("send: starting for node %s, addr %s, data len %d", node, addr, len(b)) localAddr := net.TCPAddr{ IP: t.localIP, @@ -211,7 +258,7 @@ func (t *tx) send(node, addr string, b []byte) { // with the null terminator // For now, just write b + null byte directly dataWithTerminator := append(b, 0x00) - + t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(b), 1, addr) if n, err := rawConn.Write(dataWithTerminator); err != nil { t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) @@ -244,12 +291,12 @@ func (t *tx) getPeerConn(node, addr string, localAddr net.TCPAddr) (net.Conn, er lock := lockI.(*sync.Mutex) lock.Lock() defer lock.Unlock() - + // Check if connection already exists (under per-peer lock) if connI, exists := t.peerConns.Load(node); exists { return connI.(net.Conn), nil } - + // Create new connection outside the global lock to avoid blocking other peers dialer := net.Dialer{ Timeout: t.timeout, @@ -259,7 +306,7 @@ func (t *tx) getPeerConn(node, addr string, localAddr net.TCPAddr) (net.Conn, er if err != nil { return nil, err } - + // Store the connection (still under per-peer lock) t.peerConns.Store(node, conn) t.log.Tracef("created new persistent connection to %s (%s)", node, addr) @@ -273,7 +320,7 @@ func (t *tx) removePeerConn(node string) { lock := lockI.(*sync.Mutex) lock.Lock() defer lock.Unlock() - + if connI, exists := t.peerConns.Load(node); exists { conn := connI.(net.Conn) conn.Close() From 4b6bbde2b8cb24a2829d0eb8d00aa7bdc35e1af1 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 12:13:30 +0200 Subject: [PATCH 17/54] Simplify hb.ucast: remove conn pool, workers manage their own connections Since we now have per-peer sender workers that serialize sends to each node, the connection pool infrastructure (peerConns, peerLocks, getPeerConn, removePeerConn) is no longer necessary. Each worker goroutine now maintains its own persistent connection to its assigned node. This is simpler and more efficient: - No need for sync.Maps to track connections per peer - No need for per-peer locks to prevent concurrent dial attempts - The worker's sequential nature guarantees connection safety - Connections are still reused across multiple sends to the same node - Connection cleanup happens automatically when the worker exits This reduces code complexity while maintaining the same functionality (persistent TCP connections) and the goroutine accumulation fix. --- daemon/hb/hbucast/hbtx.go | 164 +++++++++++++------------------------- 1 file changed, 55 insertions(+), 109 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index e2678c981..12b4da1f1 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -35,10 +35,6 @@ type ( cmdC chan<- interface{} msgC chan<- *hbtype.Msg cancel func() - // Per-peer connections that are kept open (sync.Map for concurrent access) - peerConns sync.Map - // Per-peer connection locks to prevent duplicate dial attempts - peerLocks sync.Map // Per-peer send queues to serialize sends to the same node sendQueues sync.Map // WaitGroup for send worker goroutines @@ -68,12 +64,6 @@ func (t *tx) Stop() error { Nodename: node, } } - // Close all peer connections - t.peerConns.Range(func(key, value any) bool { - conn := value.(net.Conn) - conn.Close() - return true - }) // Close all send queues to unblock workers t.sendQueues.Range(func(key, value any) bool { q := value.(chan sendRequest) @@ -118,8 +108,62 @@ func (t *tx) sendToNode(node, addr string, b []byte) { t.sendWorkers.Add(1) go func(node, addr string, q chan sendRequest) { defer t.sendWorkers.Done() + // Worker maintains its own persistent connection + var conn net.Conn + for req := range q { - t.doSend(req.node, req.addr, req.data) + if conn == nil { + // Create new connection + localAddr := net.TCPAddr{ + IP: t.localIP, + Port: 0, + } + dialer := net.Dialer{ + Timeout: t.timeout, + LocalAddr: &localAddr, + } + var err error + conn, err = dialer.Dial("tcp", addr) + if err != nil { + t.handleSendError(node, err) + continue + } + } + + // Set deadline on the connection + if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { + t.handleSendError(node, err) + conn.Close() + conn = nil + continue + } + + // Send the data with null terminator + dataWithTerminator := append(req.data, 0x00) + t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(req.data), 1, addr) + if n, err := conn.Write(dataWithTerminator); err != nil { + t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) + t.handleSendError(node, err) + conn.Close() + conn = nil + } else if n != len(dataWithTerminator) { + t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(dataWithTerminator)) + t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(dataWithTerminator))) + conn.Close() + conn = nil + } else { + t.log.Tracef("successfully sent %d bytes to %s", len(req.data), addr) + t.clearDedupLog(node) + t.cmdC <- hbctrl.CmdSetPeerSuccess{ + Nodename: node, + HbID: t.id, + Success: true, + } + } + } + // Close connection when worker exits + if conn != nil { + conn.Close() } }(node, addr, queue) } @@ -231,104 +275,6 @@ func (t *tx) defaultLocalIP() (net.IP, error) { return addrs[0].IP, nil } -func (t *tx) doSend(node, addr string, b []byte) { - t.log.Tracef("send: starting for node %s, addr %s, data len %d", node, addr, len(b)) - localAddr := net.TCPAddr{ - IP: t.localIP, - Port: 0, - } - - // Get or create persistent connection for this peer - rawConn, err := t.getPeerConn(node, addr, localAddr) - if err != nil { - t.handleSendError(node, err) - return - } - - // Set deadline on the raw connection - if err := rawConn.SetDeadline(time.Now().Add(t.timeout)); err != nil { - t.handleSendError(node, err) - return - } - - // Create encryptconn wrapper to add null terminator - // The data in b is already encrypted by the main hb code, but we need to add - // the null terminator that encryptconn.ReadWithNode expects - // However, b is already the final encrypted message, so we need to write it - // with the null terminator - // For now, just write b + null byte directly - dataWithTerminator := append(b, 0x00) - - t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(b), 1, addr) - if n, err := rawConn.Write(dataWithTerminator); err != nil { - t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) - t.handleSendError(node, err) - // Remove the connection so we can reconnect next time - t.removePeerConn(node) - return - } else if n != len(dataWithTerminator) { - t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(dataWithTerminator)) - t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(dataWithTerminator))) - // Remove the connection so we can reconnect next time - t.removePeerConn(node) - return - } - - t.log.Tracef("successfully sent %d bytes to %s", len(b), addr) - t.clearDedupLog(node) - - t.cmdC <- hbctrl.CmdSetPeerSuccess{ - Nodename: node, - HbID: t.id, - Success: true, - } -} - -// getPeerConn returns the persistent connection for a peer, creating it if necessary -func (t *tx) getPeerConn(node, addr string, localAddr net.TCPAddr) (net.Conn, error) { - // Get or create per-peer lock using sync.Map - lockI, _ := t.peerLocks.LoadOrStore(node, &sync.Mutex{}) - lock := lockI.(*sync.Mutex) - lock.Lock() - defer lock.Unlock() - - // Check if connection already exists (under per-peer lock) - if connI, exists := t.peerConns.Load(node); exists { - return connI.(net.Conn), nil - } - - // Create new connection outside the global lock to avoid blocking other peers - dialer := net.Dialer{ - Timeout: t.timeout, - LocalAddr: &localAddr, - } - conn, err := dialer.Dial("tcp", addr) - if err != nil { - return nil, err - } - - // Store the connection (still under per-peer lock) - t.peerConns.Store(node, conn) - t.log.Tracef("created new persistent connection to %s (%s)", node, addr) - return conn, nil -} - -// removePeerConn removes a peer connection from the map -func (t *tx) removePeerConn(node string) { - // Use per-peer lock to avoid blocking other peers - lockI, _ := t.peerLocks.LoadOrStore(node, &sync.Mutex{}) - lock := lockI.(*sync.Mutex) - lock.Lock() - defer lock.Unlock() - - if connI, exists := t.peerConns.Load(node); exists { - conn := connI.(net.Conn) - conn.Close() - t.peerConns.Delete(node) - t.log.Tracef("removed connection to %s", node) - } -} - // handleSendError handles send errors with deduplication logging func (t *tx) handleSendError(node string, err error) { newErr := err.Error() From 9b5aed923e5ab097f457de634767178b9fe9dd49 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 12:28:01 +0200 Subject: [PATCH 18/54] Fix frame loss in encryptconn: use persistent scanner per connection The getMessage function was creating a new bufio.Scanner for each ReadWithNode call. When TCP coalesces multiple authenticated writes, the scanner's internal buffer can contain additional frames after the first NUL delimiter. Discarding the scanner after reading the first frame loses these buffered bytes, causing valid heartbeat/state messages to be silently dropped. This change makes the scanner persistent per connection: - Store scanner and its buffer in the T struct - Initialize scanner once when creating the T connection wrapper - Reuse the same scanner for all ReadWithNode calls on that connection - Return buffer to pool when connection is closed This preserves the existing NUL framing and ensures all frames are read, even when TCP coalesces multiple writes. Each frame is still decrypted independently after being read from the scanner. --- daemon/encryptconn/main.go | 39 ++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/daemon/encryptconn/main.go b/daemon/encryptconn/main.go index 21e050505..add6f898e 100644 --- a/daemon/encryptconn/main.go +++ b/daemon/encryptconn/main.go @@ -6,7 +6,6 @@ package encryptconn import ( "bufio" "bytes" - "io" "net" "sync" ) @@ -24,6 +23,9 @@ type ( // srcNode is the encrypter nodename returned by ReadWithNode srcNode string encryptDecrypter encryptDecrypter + // Persistent scanner for reading NUL-delimited frames + scanner *bufio.Scanner + scannerBuf []byte } ConnNoder interface { @@ -49,10 +51,15 @@ var ( // New returns a new *T that will use encrypted net.Conn func New(encConn net.Conn, ed encryptDecrypter) *T { - return &T{ + t := &T{ Conn: encConn, encryptDecrypter: ed, + scannerBuf: msgPool.Get().([]byte), } + t.scanner = bufio.NewScanner(encConn) + t.scanner.Buffer(t.scannerBuf, msgMaxSize) + t.scanner.Split(splitFunc) + return t } // Write implement Writer interface for T @@ -80,7 +87,7 @@ func (t *T) Read(b []byte) (n int, err error) { // read and decrypt data read from t.Conn func (t *T) ReadWithNode(b []byte) (n int, nodename string, err error) { var encBytes, clearBytes []byte - if encBytes, err = getMessage(t.Conn); err != nil { + if encBytes, err = t.getMessage(); err != nil { return } if clearBytes, nodename, err = t.encryptDecrypter.DecryptWithNode(encBytes); err != nil { @@ -95,6 +102,16 @@ func (t *T) SrcNode() string { return t.srcNode } +// Close closes the underlying connection and releases the scanner buffer +func (t *T) Close() error { + if t.scannerBuf != nil { + msgPool.Put(t.scannerBuf) + t.scannerBuf = nil + t.scanner = nil + } + return t.Conn.Close() +} + // dropCR drops a terminal \r from the data. func dropCR(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\r' { @@ -121,15 +138,13 @@ func splitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { return 0, nil, nil } -func getMessage(r io.Reader) ([]byte, error) { - scanner := bufio.NewScanner(r) - sharedBuffer := msgPool.Get().([]byte) - defer func() { msgPool.Put(sharedBuffer) }() - scanner.Buffer(sharedBuffer, msgMaxSize) - scanner.Split(splitFunc) - scanner.Scan() - sharedB := scanner.Bytes() +// getMessage reads a single NUL-delimited frame from the persistent scanner +func (t *T) getMessage() ([]byte, error) { + if !t.scanner.Scan() { + return nil, t.scanner.Err() + } + sharedB := t.scanner.Bytes() b := make([]byte, len(sharedB)) copy(b, sharedB) - return b, scanner.Err() + return b, nil } From fb8c9627cf2678bdf31576e5f9f948216e8cbac0 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 13:49:28 +0200 Subject: [PATCH 19/54] Fix hb.ucast shutdown: make workers cancellation-aware The previous implementation had a shutdown race condition: - Stop() closes send queues and waits for workers via sendWorkers.Wait() - But workers blocked in Dial() or Write() would not be interrupted - Workers could also block indefinitely on cmdC send after controller stopped This fix makes workers cancellation-aware: 1. Worker loop now selects on t.ctx.Done() as well as the queue 2. Dial uses DialContext with a timeout that respects t.ctx 3. cmdC send is wrapped in a select that also checks t.ctx.Done() 4. On context cancellation, workers close their connection and exit immediately This ensures that: - Workers are interrupted when the transmitter is stopped - sendWorkers.Wait() completes promptly (within timeout or immediately on cancel) - Heartbeat shutdown is fast and won't hang on slow/unreachable peers - Post-send controller notifications don't block after controller exits --- daemon/hb/hbucast/hbtx.go | 127 ++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 47 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 12b4da1f1..ff248efb0 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -110,61 +110,94 @@ func (t *tx) sendToNode(node, addr string, b []byte) { defer t.sendWorkers.Done() // Worker maintains its own persistent connection var conn net.Conn - - for req := range q { - if conn == nil { - // Create new connection - localAddr := net.TCPAddr{ - IP: t.localIP, - Port: 0, + localAddr := net.TCPAddr{ + IP: t.localIP, + Port: 0, + } + + for { + select { + case <-t.ctx.Done(): + // Context cancelled, close connection and exit + if conn != nil { + conn.Close() + conn = nil + } + return + case req, ok := <-q: + if !ok { + // Queue closed, exit + if conn != nil { + conn.Close() + } + return } - dialer := net.Dialer{ - Timeout: t.timeout, - LocalAddr: &localAddr, + + if conn == nil { + // Create new connection with context-aware dialer + dialer := &net.Dialer{ + Timeout: t.timeout, + LocalAddr: &localAddr, + } + // Use a separate context for dial that respects t.ctx + dialCtx, dialCancel := context.WithTimeout(t.ctx, t.timeout) + defer dialCancel() + var err error + conn, err = dialer.DialContext(dialCtx, "tcp", addr) + if err != nil { + t.handleSendError(node, err) + continue + } } - var err error - conn, err = dialer.Dial("tcp", addr) - if err != nil { + + // Set deadline on the connection + if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { t.handleSendError(node, err) + conn.Close() + conn = nil continue } - } - - // Set deadline on the connection - if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { - t.handleSendError(node, err) - conn.Close() - conn = nil - continue - } - - // Send the data with null terminator - dataWithTerminator := append(req.data, 0x00) - t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(req.data), 1, addr) - if n, err := conn.Write(dataWithTerminator); err != nil { - t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) - t.handleSendError(node, err) - conn.Close() - conn = nil - } else if n != len(dataWithTerminator) { - t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(dataWithTerminator)) - t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(dataWithTerminator))) - conn.Close() - conn = nil - } else { - t.log.Tracef("successfully sent %d bytes to %s", len(req.data), addr) - t.clearDedupLog(node) - t.cmdC <- hbctrl.CmdSetPeerSuccess{ - Nodename: node, - HbID: t.id, - Success: true, + + // Send the data with null terminator + dataWithTerminator := append(req.data, 0x00) + t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(req.data), 1, addr) + if n, err := conn.Write(dataWithTerminator); err != nil { + t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) + t.handleSendError(node, err) + conn.Close() + conn = nil + } else if n != len(dataWithTerminator) { + t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(dataWithTerminator)) + t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(dataWithTerminator))) + conn.Close() + conn = nil + } else { + t.log.Tracef("successfully sent %d bytes to %s", len(req.data), addr) + t.clearDedupLog(node) + // Send success notification, but don't block on it + select { + case t.cmdC <- hbctrl.CmdSetPeerSuccess{ + Nodename: node, + HbID: t.id, + Success: true, + }: + case <-t.ctx.Done(): + // Context cancelled, skip notification + if conn != nil { + conn.Close() + conn = nil + } + return + } + + // Reset deadline for next write (connection stays open) + if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { + t.log.Tracef("failed to reset deadline for %s: %v", addr, err) + // Continue with connection, it might still work + } } } } - // Close connection when worker exits - if conn != nil { - conn.Close() - } }(node, addr, queue) } From 1d4cfaeeebd61813452331c3410a3cfff05c503e Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 14:45:47 +0200 Subject: [PATCH 20/54] Fix hb.ucast rx resource exhaustion: one handler per peer, replace conn on reconnect Instead of allowing unlimited connections from each peer IP, enforce one handler goroutine per peer. When a peer reconnects: - Close the old connection (old handler exits on next read) - Accept the new connection and start a new handler - Peer doesn't see errors or retry The accept loop (single-threaded) manages a peerConns map: - On new connection: check if peer has entry, close old conn if so - Store new connection and start handler - Handlers don't touch the map; only read their own connection Combined with read deadline (3x timeout, min 10s), this ensures: - Bounded goroutines (one per cluster peer) - Bounded memory (10MB per peer) - Seamless connection replacement on reconnect - No retry storms from peers --- daemon/hb/hbucast/hbrx.go | 89 +++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 48acd3ccb..936a51d95 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -36,6 +36,10 @@ type ( cmdC chan<- interface{} msgC chan<- *hbtype.Msg cancel func() + + // Track current connection per peer (peerAddr -> encryptconn.ConnNoder) + // Accept loop is the only writer; handlers only read their own connection + peerConns sync.Map } ) @@ -67,6 +71,9 @@ func (t *rx) Stop() error { Nodename: node, } } + // Note: We don't close active connections here. The handlers will exit + // when their read deadline expires or when the peer closes the connection. + // The listener is closed which prevents new connections. t.Wait() t.log.Tracef("wait done") return nil @@ -192,21 +199,33 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { connAddr, _, err := net.SplitHostPort(conn.RemoteAddr().String()) if err != nil { t.log.Warnf("%s", err) + conn.Close() continue } if _, ok := otherNodeIPM[connAddr]; !ok { t.log.Warnf("unexpected connection from %s", connAddr) - if err := conn.Close(); err != nil { - t.log.Warnf("failed to close unexpected connection from %s: %s", connAddr, err) - } + conn.Close() continue } clearConn := encryptconn.New(conn, crypto.Load()) + // Check if we already have a handler for this peer and close its connection + // This is done atomically in the accept loop before starting new handler + if oldConnI, hasOld := t.peerConns.Load(connAddr); hasOld { + oldConn := oldConnI.(encryptconn.ConnNoder) + t.log.Tracef("replacing existing connection from %s with new one", connAddr) + // Close old connection; its handler will exit on next read with error + oldConn.Close() + t.peerConns.Delete(connAddr) + } + // Store new connection before starting handler to prevent race + t.peerConns.Store(connAddr, clearConn) wg.Add(1) - go func() { + go func(peerAddr string, c encryptconn.ConnNoder) { defer wg.Done() - t.handleLoop(clearConn) - }() + defer c.Close() + // Do NOT touch peerConns map - only accept loop manages it + t.handleLoop(c, peerAddr) + }(connAddr, clearConn) } wg.Wait() t.log.Infof("stopped %s", t.addr) @@ -216,53 +235,61 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { return nil } -func (t *rx) handleLoop(conn encryptconn.ConnNoder) { - // Read messages in a loop to support connection reuse from pool on sender side - // When sender uses connection pooling, multiple messages are sent on the same connection. - // We don't set a read deadline here to avoid timing out on idle connections between heartbeats. - // The connection will be closed by the sender when it's done. - defer func() { - t.log.Tracef("handleLoop: connection closed from %s", conn.RemoteAddr()) - if err := conn.Close(); err != nil { - t.log.Warnf("unexpected error while closing connection from %s: %s", conn.RemoteAddr(), err) - } - }() - - t.log.Tracef("handleLoop: starting to read messages from %s", conn.RemoteAddr()) - +func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { + // Set a generous read deadline to prevent idle connections from blocking indefinitely. + // This is long enough to cover the heartbeat interval (default 5s) with some margin. + // The deadline will be reset before each read operation. + deadline := t.timeout * 3 + if deadline < 10*time.Second { + deadline = 10 * time.Second + } + t.log.Tracef("handleLoop: starting to read messages from %s", peerAddr) data := msgPool.Get().([]byte) defer func() { msgPool.Put(data) }() - + msgCount := 0 for { + // Check context before blocking on read + select { + case <-t.ctx.Done(): + t.log.Tracef("handleLoop: context cancelled, stopping after %d messages from %s", msgCount, peerAddr) + return + default: + } + + // Set read deadline for this iteration + if err := conn.SetReadDeadline(time.Now().Add(deadline)); err != nil { + t.log.Warnf("handleLoop: failed to set read deadline for %s: %v", peerAddr, err) + return + } + // Reset buffer for each message buffer := data[:cap(data)] - - // No deadline - connection is kept open by sender with persistent connections - // Read will block until data arrives or connection is closed + + // Read will block until data arrives, connection is closed, or deadline is reached i, nodename, err := conn.ReadWithNode(buffer) if err != nil { if err.Error() != "EOF" { - t.log.Tracef("handleLoop: read failed from %s after %d messages: %s", conn.RemoteAddr(), msgCount, err) + t.log.Tracef("handleLoop: read failed from %s after %d messages: %s", peerAddr, msgCount, err) } else { - t.log.Tracef("handleLoop: EOF from %s after %d messages", conn.RemoteAddr(), msgCount) + t.log.Tracef("handleLoop: EOF from %s after %d messages", peerAddr, msgCount) } return } msgCount++ - + t.log.Tracef("handleLoop: received %d bytes from node %s (msg #%d)", i, nodename, msgCount) - + if i >= (msgMaxSize - 10000) { - t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, conn.RemoteAddr(), i) + t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, peerAddr, i) } msg := hbtype.Msg{} if err := json.Unmarshal(buffer[:i], &msg); err != nil { - t.log.Warnf("handleLoop: unmarshal message failed from node %s:%s: %s", nodename, conn.RemoteAddr(), err) + t.log.Warnf("handleLoop: unmarshal message failed from node %s:%s: %s", nodename, peerAddr, err) return } t.log.Tracef("handleLoop: successfully decoded message from node %s (kind=%s)", nodename, msg.Kind) - + cmdPeerSuccess := hbctrl.CmdSetPeerSuccess{ Nodename: msg.Nodename, HbID: t.id, From d0bba9d4493a0536c9c62a96fe8b0c27db5e9bb8 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 15:28:51 +0200 Subject: [PATCH 21/54] Fix encryptconn scanner buffer recycling race: don't pool the buffer The persistent scanner introduced with the per-connection scanner change brought a Close() that returned the scanner buffer to msgPool and nilled the scanner. Both are unsafe while a reader goroutine is still using the connection, and hb.ucast rx has exactly that pattern: the accept loop calls oldConn.Close() when a peer reconnects, while the old handler goroutine is still in its read loop (and the handler itself closes the connection again on exit). Two failure modes: - Nil dereference: a handler parked on the msgC send resumes after Close() has set t.scanner to nil, then calls scanner.Scan() and panics the daemon. - Cross-connection buffer sharing: the "scannerBuf != nil" check is not atomic, so both closers can Put the same 10MB slice back to the pool. Two later connections then get the same backing array and their scanners write into it, corrupting frames and failing decryption. Give the buffer to the *T for the lifetime of the connection instead: - Remove msgPool and the scannerBuf field - Remove the Close() override, so Close() resolves to the embedded net.Conn, which is race-free and safe to call twice - Size the buffer at msgUsualSize and let bufio.Scanner grow it, up to msgMaxSize, only when a bigger message is read Dropping the pool loses buffer reuse across connections, but not memory: the pooled buffer was held for the whole connection lifetime anyway. In hb.ucast connections are one per peer and long-lived, so the allocation churn is negligible, and the steady-state footprint per connection drops from 10MB to msgUsualSize. --- daemon/encryptconn/main.go | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/daemon/encryptconn/main.go b/daemon/encryptconn/main.go index add6f898e..c30042c7e 100644 --- a/daemon/encryptconn/main.go +++ b/daemon/encryptconn/main.go @@ -7,7 +7,6 @@ import ( "bufio" "bytes" "net" - "sync" ) type ( @@ -24,8 +23,7 @@ type ( srcNode string encryptDecrypter encryptDecrypter // Persistent scanner for reading NUL-delimited frames - scanner *bufio.Scanner - scannerBuf []byte + scanner *bufio.Scanner } ConnNoder interface { @@ -38,26 +36,22 @@ var ( msgUsualSize = 1000 // usual event size msgMaxSize = 10000000 // max kind=full event size - - // Create a new sync.Pool to manage the byte buffers. Used to reduce memory usage - // when many messages are scanned. - msgPool = sync.Pool{ - New: func() interface{} { - // This creates a new byte slice of the specified size. - return make([]byte, msgMaxSize) - }, - } ) // New returns a new *T that will use encrypted net.Conn +// +// The scanner buffer is owned by the returned *T for its whole lifetime: it is +// not pooled, so that a Close() concurrent with a reader can't hand the buffer +// to another connection while the scanner still points into it. It starts at +// the usual message size and is grown by the scanner, up to the max message +// size, when a bigger message is read. func New(encConn net.Conn, ed encryptDecrypter) *T { t := &T{ Conn: encConn, encryptDecrypter: ed, - scannerBuf: msgPool.Get().([]byte), } t.scanner = bufio.NewScanner(encConn) - t.scanner.Buffer(t.scannerBuf, msgMaxSize) + t.scanner.Buffer(make([]byte, msgUsualSize), msgMaxSize) t.scanner.Split(splitFunc) return t } @@ -102,16 +96,6 @@ func (t *T) SrcNode() string { return t.srcNode } -// Close closes the underlying connection and releases the scanner buffer -func (t *T) Close() error { - if t.scannerBuf != nil { - msgPool.Put(t.scannerBuf) - t.scannerBuf = nil - t.scanner = nil - } - return t.Conn.Close() -} - // dropCR drops a terminal \r from the data. func dropCR(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\r' { From 18d48234cf2681894fd0fbfa5193575a038cbe4b Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 15:29:42 +0200 Subject: [PATCH 22/54] Fix hb.ucast tx Stop panic on send to closed queue Stop() closed every per-node send queue before waiting on the Start goroutine. That goroutine is the only sendToNode caller, and it is still running at that point: if it had already passed its select on msgC when the context was cancelled, it goes on to send on a queue Stop() just closed, panicking the daemon. The CmdDelWatcher sends in between make the window comfortably wide. Wait for the Start goroutine first, so the queues provably have no writer left, then close them. Also delete each queue from sendQueues before closing it, so a second Stop() doesn't close an already closed channel: a tx can be stopped both by the daemonctl path and by the config change path. Worst case Stop() still waits up to timeout for a worker parked in conn.Write. Closing the queues never unblocked a write either. --- daemon/hb/hbucast/hbtx.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index ff248efb0..587789877 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -64,13 +64,17 @@ func (t *tx) Stop() error { Nodename: node, } } + // Wait for the Start goroutine first: it is the only sendToNode caller, + // so the send queues have no writer left once it is done. Closing them + // before would risk a send on a closed channel. + t.Wait() // Close all send queues to unblock workers t.sendQueues.Range(func(key, value any) bool { q := value.(chan sendRequest) + t.sendQueues.Delete(key) close(q) return true }) - t.Wait() t.sendWorkers.Wait() t.log.Tracef("wait done") return nil From c063b3459b8f26b7c4b604d5fb27f86af5446664 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 15:35:22 +0200 Subject: [PATCH 23/54] Don't defer the dial cancel in the hb.ucast tx worker loop The send worker dials inside its loop and deferred the context cancel. The worker goroutine lives as long as the transmitter, so those deferred cancels are never run until stop: every reconnect leaves another one on the goroutine stack, growing without bound on a flapping peer. Cancel right after the dial returns instead. Cancelling the dial context once the connection is established doesn't affect the connection, so the dial deadline still applies and nothing else changes. --- daemon/hb/hbucast/hbtx.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 587789877..91e9f498d 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -143,15 +143,20 @@ func (t *tx) sendToNode(node, addr string, b []byte) { Timeout: t.timeout, LocalAddr: &localAddr, } - // Use a separate context for dial that respects t.ctx + // Use a separate context for dial that respects t.ctx. + // Cancel as soon as the dial returns: cancelling after a + // successful dial doesn't affect the connection, and this + // worker goroutine lives as long as the transmitter, so a + // deferred cancel would pile up on its stack, one per + // reconnect. dialCtx, dialCancel := context.WithTimeout(t.ctx, t.timeout) - defer dialCancel() - var err error - conn, err = dialer.DialContext(dialCtx, "tcp", addr) + newConn, err := dialer.DialContext(dialCtx, "tcp", addr) + dialCancel() if err != nil { t.handleSendError(node, err) continue } + conn = newConn } // Set deadline on the connection From 0eeaf009c1796935e4e951bc2bc4f1377e4d8176 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 15:41:58 +0200 Subject: [PATCH 24/54] Fix hb.ucast tx localIP data race, and pick up local ip changes Each send worker read t.localIP from its own goroutine to build the source address it dials from, while the Start goroutine refreshes t.localIP every 30s: a data race on the net.IP, and the refresh never reached an already running worker, which kept binding the source address it snapshotted when it was created. Carry the local ip in the sendRequest instead. The Start goroutine is the only sendToNode caller, so t.localIP is now read and written by that goroutine only, and the value the worker dials from is the one current when the message was queued. A worker whose connection is bound to a source address that is no longer the local ip now closes it and redials, instead of waiting for the write to fail once the address is gone from the interface. --- daemon/hb/hbucast/hbtx.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 91e9f498d..919dac5de 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -47,6 +47,11 @@ type sendRequest struct { node string addr string data []byte + + // localIP is the source address the worker must dial from. It is + // carried by the request because t.localIP is refreshed by the Start + // goroutine, which is the only one allowed to read or write it. + localIP net.IP } // ID implements the ID function of Transmitter interface for tx @@ -102,6 +107,8 @@ func (t *tx) streamPeerDesc(addr string) string { // sendToNode queues a send request for a specific node // It creates a worker goroutine for the node if one doesn't exist +// +// Must be called from the Start goroutine: it reads t.localIP. func (t *tx) sendToNode(node, addr string, b []byte) { // Get or create send queue for this node queueI, loaded := t.sendQueues.LoadOrStore(node, make(chan sendRequest, 1)) @@ -114,10 +121,9 @@ func (t *tx) sendToNode(node, addr string, b []byte) { defer t.sendWorkers.Done() // Worker maintains its own persistent connection var conn net.Conn - localAddr := net.TCPAddr{ - IP: t.localIP, - Port: 0, - } + // connLocalIP is the source address conn is bound to, to + // detect a local ip change while conn is established + var connLocalIP net.IP for { select { @@ -137,8 +143,21 @@ func (t *tx) sendToNode(node, addr string, b []byte) { return } + if conn != nil && !connLocalIP.Equal(req.localIP) { + // The local ip changed since we dialed: the + // connection is bound to an address the node may + // not own anymore, redial from the new one. + t.log.Infof("local ip changed from %s to %s, reconnect to %s", connLocalIP, req.localIP, addr) + conn.Close() + conn = nil + } + if conn == nil { // Create new connection with context-aware dialer + localAddr := net.TCPAddr{ + IP: req.localIP, + Port: 0, + } dialer := &net.Dialer{ Timeout: t.timeout, LocalAddr: &localAddr, @@ -157,6 +176,7 @@ func (t *tx) sendToNode(node, addr string, b []byte) { continue } conn = newConn + connLocalIP = req.localIP } // Set deadline on the connection @@ -212,7 +232,7 @@ func (t *tx) sendToNode(node, addr string, b []byte) { // Try to send without blocking first (non-blocking send) select { - case queue <- sendRequest{node: node, addr: addr, data: b}: + case queue <- sendRequest{node: node, addr: addr, data: b, localIP: t.localIP}: // Successfully queued default: // Queue is full, drop the message to avoid blocking From 344b5adb08673c21798ff4546c15ff61e530a993 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 15:47:55 +0200 Subject: [PATCH 25/54] Close hb.ucast rx peer connections when the accept loop stops Stop() cancelled the context and waited, but left the established peer connections open, so a handler parked in ReadWithNode held the shutdown until its read deadline expired: up to 3x timeout, 45s with the defaults. A peer that keeps beating wakes its handler within an interval, but a silent or partitioned peer didn't, and stopping or reconfiguring the heartbeat stalled for that long. Close the tracked connections at the end of the accept loop, right before waiting for the handlers. Doing it there rather than in Stop() keeps the invariant that only the accept loop writes peerConns: the loop has stopped accepting by then, so no connection can be added behind our back, and closing a connection unblocks the read its handler is in. --- daemon/hb/hbucast/hbrx.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 936a51d95..85aae7d1a 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -71,9 +71,9 @@ func (t *rx) Stop() error { Nodename: node, } } - // Note: We don't close active connections here. The handlers will exit - // when their read deadline expires or when the peer closes the connection. - // The listener is closed which prevents new connections. + // Note: the active connections are closed by the accept loop when it + // stops, so a handler blocked in ReadWithNode doesn't hold the shutdown + // until its read deadline expires. t.Wait() t.log.Tracef("wait done") return nil @@ -227,6 +227,15 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { t.handleLoop(c, peerAddr) }(connAddr, clearConn) } + // The accept loop is the only writer of peerConns and it is done: + // no new connection can show up. Close the ones still tracked to + // unblock their handler, which would otherwise sit in ReadWithNode + // until its read deadline expires. + t.peerConns.Range(func(key, value any) bool { + t.peerConns.Delete(key) + _ = value.(encryptconn.ConnNoder).Close() + return true + }) wg.Wait() t.log.Infof("stopped %s", t.addr) }() From c8683732f5f77e0acdfc9959a8d53f873b58daab Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 16:05:13 +0200 Subject: [PATCH 26/54] Maintain the dns name index incrementally rebuildNameIndex() reallocated the whole name to records index from all of t.state, and was called on every InstanceStatusUpdated, one of the highest frequency events in the daemon, even when the handler had published no record change at all. That traded a per lookup cost for a per event cost that grows with the number of objects in the cluster. Keep the index in sync where the state changes instead: - setStateRecords() replaces the records of one state key, unindexing the ones it had and indexing the new ones, so the cost is scoped to the object that changed - setClusterRecords() refreshes the SOA and NS records only, on cluster config change - delIndexRecord() removes a single occurrence: the same record can be indexed once per state key that yields it, and the others must stay Record.Key() replaces the recordKey literals built at each site, and the cluster records construction is now shared by the index and zone(), which had two copies of it. TestNameIndexIsMaintainedIncrementally checks the invariant the full rebuild gave for free: after any sequence of state and cluster config changes, the index holds exactly the records a rebuild from scratch would, duplicates included. --- daemon/dns/main.go | 11 ++- daemon/dns/main_cmd.go | 155 ++++++++++++++++++++---------------- daemon/dns/main_cmd_test.go | 127 +++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 71 deletions(-) create mode 100644 daemon/dns/main_cmd_test.go diff --git a/daemon/dns/main.go b/daemon/dns/main.go index 3e746b4a9..45026a4a7 100644 --- a/daemon/dns/main.go +++ b/daemon/dns/main.go @@ -46,9 +46,16 @@ type ( // The zone data is obtained by merging all map values. state map[stateKey]map[recordKey]Record - // nameIndex maps record names to their records for O(1) lookups + // nameIndex maps record names to their records for O(1) lookups. + // It is maintained incrementally: a state change only reindexes + // the records of the state key it changes, and a cluster config + // change only the cluster records. nameIndex map[string][]Record + // clusterRecords is the cluster level records (zone SOA, nameservers + // NS and A) currently indexed in nameIndex. + clusterRecords Zone + // score stores the node.Stats.Score values, to use as weight in SRV records score map[string]int @@ -128,7 +135,7 @@ func (t *Manager) Start(parent context.Context) error { t.startSubscriptions() t.clusterConfig = *cluster.ConfigData.Get() - t.rebuildNameIndex() + t.setClusterRecords() if err := t.startUDSListener(); err != nil { return err diff --git a/daemon/dns/main_cmd.go b/daemon/dns/main_cmd.go index aefc81db2..a4457e6d3 100644 --- a/daemon/dns/main_cmd.go +++ b/daemon/dns/main_cmd.go @@ -46,6 +46,12 @@ type recordKey struct { Content string } +// Key returns the identity of the record, the part of it that TTL and +// DomainID changes don't affect. +func (t Record) Key() recordKey { + return recordKey{t.Name, t.Type, t.Content} +} + func (t *Manager) onNodeStatsUpdated(c *msgbus.NodeStatsUpdated) { t.score[c.Node] = c.Value.Score } @@ -73,8 +79,8 @@ func (t *Manager) onClusterConfigUpdated(c *msgbus.ClusterConfigUpdated) { if change { t.publishSubsystemDnsUpdated() } - // Rebuild nameIndex since SOA/NS records depend on clusterConfig.DNS - t.rebuildNameIndex() + // Refresh the indexed SOA/NS records, they depend on clusterConfig.DNS + t.setClusterRecords() } func (t *Manager) pubDeleted(record Record, p naming.Path, node string) { @@ -105,10 +111,8 @@ func (t *Manager) onInstanceStatusDeleted(c *msgbus.InstanceStatusDeleted) { for _, record := range recordMap { t.pubDeleted(record, c.Path, c.Node) } - delete(t.state, key) + t.setStateRecords(key, nil) } - // Rebuild nameIndex for O(1) lookups - t.rebuildNameIndex() } func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { @@ -119,10 +123,10 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { existingRecordsMap := t.state[key] stage := func(record Record) { - recordKey := recordKey{record.Name, record.Type, record.Content} + recKey := record.Key() // Check if this record already exists (by identity, not by TTL/DomainID) - if existingRecord, ok := existingRecordsMap[recordKey]; !ok { + if existingRecord, ok := existingRecordsMap[recKey]; !ok { // New record, publish update t.pubUpdated(record, c.Path, c.Node) } else if existingRecord != record { @@ -130,7 +134,7 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { t.pubUpdated(record, c.Path, c.Node) } // Store in new records map (preserves the full Record with current TTL/DomainID) - newRecordsMap[recordKey] = record + newRecordsMap[recKey] = record } stageSRV := func(s string) error { expose, err := ParseExpose(s) @@ -291,14 +295,7 @@ func (t *Manager) onInstanceStatusUpdated(c *msgbus.InstanceStatusUpdated) { } } - // Update state: if we have new records, store them; otherwise clean up - if len(newRecordsMap) > 0 { - t.state[key] = newRecordsMap - } else { - delete(t.state, key) - } - // Rebuild nameIndex for O(1) lookups - t.rebuildNameIndex() + t.setStateRecords(key, newRecordsMap) } func (t *Manager) onCmdGet(c cmdGet) { @@ -316,7 +313,7 @@ func (t *Manager) onCmdGet(c cmdGet) { if (c.Type != "ANY") && (record.Type != c.Type) { continue } - key := recordKey{record.Name, record.Type, record.Content} + key := record.Key() if !seen[key] { zone = append(zone, record) seen[key] = true @@ -332,6 +329,23 @@ func (t *Manager) onCmdGetZone(c cmdGetZone) { } func (t *Manager) zone() Zone { + zone := t.clusterRecordZone() + seen := make(map[recordKey]bool) + for _, recordMap := range t.state { + for _, record := range recordMap { + key := record.Key() + if !seen[key] { + zone = append(zone, record) + seen[key] = true + } + } + } + return zone +} + +// clusterRecordZone returns the cluster level records: the zone SOA, and +// the NS and A records of each configured nameserver. +func (t *Manager) clusterRecordZone() Zone { zone := make(Zone, 0) zoneName := t.clusterConfig.Name + "." for i, dns := range t.clusterConfig.DNS { @@ -361,63 +375,66 @@ func (t *Manager) zone() Zone { }, ) } - seen := make(map[recordKey]bool) - for _, recordMap := range t.state { - for _, record := range recordMap { - key := recordKey{record.Name, record.Type, record.Content} - if !seen[key] { - zone = append(zone, record) - seen[key] = true - } - } - } return zone } -// rebuildNameIndex rebuilds the name-to-records index including both -// state records and cluster-level SOA/NS records -func (t *Manager) rebuildNameIndex() { - t.nameIndex = make(map[string][]Record) - - // Add cluster-level SOA and NS records - zoneName := t.clusterConfig.Name + "." - for i, dns := range t.clusterConfig.DNS { - nsName := fmt.Sprintf("ns%d.%s", i+1, zoneName) - soaContent := fmt.Sprintf("dns.%s %s %d %d %d %d %d", zoneName, contact, serial, refresh, retry, expire, minimum) - - // SOA record - t.nameIndex[zoneName] = append(t.nameIndex[zoneName], Record{ - Name: zoneName, - DomainID: -1, - Type: "SOA", - TTL: 60, - Content: soaContent, - }) - - // A record for nameserver - t.nameIndex[nsName] = append(t.nameIndex[nsName], Record{ - Name: nsName, - DomainID: -1, - Type: "A", - TTL: 60, - Content: dns, - }) - - // NS record - t.nameIndex[zoneName] = append(t.nameIndex[zoneName], Record{ - Name: zoneName, - DomainID: -1, - Type: "NS", - TTL: 3600, - Content: nsName, - }) +// setClusterRecords replaces the cluster level records in the name index +// with the ones the current cluster config yields. +func (t *Manager) setClusterRecords() { + for _, record := range t.clusterRecords { + t.delIndexRecord(record) } - - // Add instance records from state - for _, recordMap := range t.state { - for _, record := range recordMap { - t.nameIndex[record.Name] = append(t.nameIndex[record.Name], record) + t.clusterRecords = t.clusterRecordZone() + for _, record := range t.clusterRecords { + t.addIndexRecord(record) + } +} + +// setStateRecords replaces the records of a state key, and keeps the name +// index in sync: only the records of this key are reindexed, so the cost +// doesn't grow with the number of objects in the cluster. +// +// A nil recordMap drops the state key. +func (t *Manager) setStateRecords(key stateKey, recordMap map[recordKey]Record) { + for _, record := range t.state[key] { + t.delIndexRecord(record) + } + for _, record := range recordMap { + t.addIndexRecord(record) + } + if len(recordMap) > 0 { + t.state[key] = recordMap + } else { + delete(t.state, key) + } +} + +// addIndexRecord adds a record to the name index +func (t *Manager) addIndexRecord(record Record) { + t.nameIndex[record.Name] = append(t.nameIndex[record.Name], record) +} + +// delIndexRecord removes one occurrence of record from the name index. +// The same record can be indexed more than once, when several state keys +// or several nameservers yield it, so the other occurrences are kept. +func (t *Manager) delIndexRecord(record Record) { + records, ok := t.nameIndex[record.Name] + if !ok { + return + } + key := record.Key() + for i, indexed := range records { + if indexed.Key() != key { + continue } + records[i] = records[len(records)-1] + records = records[:len(records)-1] + if len(records) == 0 { + delete(t.nameIndex, record.Name) + } else { + t.nameIndex[record.Name] = records + } + return } } diff --git a/daemon/dns/main_cmd_test.go b/daemon/dns/main_cmd_test.go new file mode 100644 index 000000000..fef0c89a1 --- /dev/null +++ b/daemon/dns/main_cmd_test.go @@ -0,0 +1,127 @@ +package dns + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opensvc/om3/v3/core/cluster" +) + +// nameIndexFromScratch builds the name index the way a full rebuild would, +// to compare it with the incrementally maintained one. +func nameIndexFromScratch(t *Manager) map[string][]Record { + index := make(map[string][]Record) + for _, record := range t.clusterRecords { + index[record.Name] = append(index[record.Name], record) + } + for _, recordMap := range t.state { + for _, record := range recordMap { + index[record.Name] = append(index[record.Name], record) + } + } + return index +} + +// requireIndexInSync verifies the name index holds, for each name, the same +// records as a full rebuild would. Both are compared as multisets: the index +// order is not significant, and a record indexed twice must stay indexed +// twice. +func requireIndexInSync(t *testing.T, m *Manager) { + t.Helper() + count := func(index map[string][]Record) map[Record]int { + counted := make(map[Record]int) + for name, records := range index { + require.NotEmpty(t, records, "name %s is indexed with no record", name) + for _, record := range records { + require.Equal(t, name, record.Name, "record indexed under the wrong name") + counted[record]++ + } + } + return counted + } + require.Equal(t, count(nameIndexFromScratch(m)), count(m.nameIndex)) +} + +func recordMapOf(records ...Record) map[recordKey]Record { + recordMap := make(map[recordKey]Record) + for _, record := range records { + recordMap[record.Key()] = record + } + return recordMap +} + +func TestNameIndexIsMaintainedIncrementally(t *testing.T) { + m := &Manager{ + state: make(map[stateKey]map[recordKey]Record), + nameIndex: make(map[string][]Record), + clusterConfig: cluster.Config{ + Name: "cluster1", + DNS: []string{"10.0.0.1", "10.0.0.2"}, + }, + } + + var ( + key1 = stateKey{path: "system/svc/svc1", node: "node1"} + key2 = stateKey{path: "system/svc/svc1", node: "node2"} + + // the fqdn record both instances of svc1 yield, indexed twice + shared = Record{Name: "svc1.system.svc.cluster1.", Type: "A", TTL: 60, Content: "10.1.1.1"} + + onNode1 = Record{Name: "svc1.system.svc.node1.cluster1.", Type: "A", TTL: 60, Content: "10.1.1.1"} + onNode2 = Record{Name: "svc1.system.svc.node2.cluster1.", Type: "A", TTL: 60, Content: "10.1.1.2"} + ) + + m.setClusterRecords() + requireIndexInSync(t, m) + + t.Run("index the records of a state key", func(t *testing.T) { + m.setStateRecords(key1, recordMapOf(shared, onNode1)) + requireIndexInSync(t, m) + }) + + t.Run("index a record another state key already yields", func(t *testing.T) { + m.setStateRecords(key2, recordMapOf(shared, onNode2)) + requireIndexInSync(t, m) + require.Len(t, m.nameIndex[shared.Name], 2, "the shared record must be indexed once per state key") + }) + + t.Run("dropping a state key keeps the records of the others", func(t *testing.T) { + m.setStateRecords(key1, nil) + requireIndexInSync(t, m) + require.NotContains(t, m.nameIndex, onNode1.Name) + require.Len(t, m.nameIndex[shared.Name], 1, "the shared record must stay indexed for the remaining key") + }) + + t.Run("changing the records of a state key", func(t *testing.T) { + changed := onNode2 + changed.Content = "10.1.1.3" + m.setStateRecords(key2, recordMapOf(shared, changed)) + requireIndexInSync(t, m) + require.Equal(t, []Record{changed}, m.nameIndex[changed.Name]) + }) + + t.Run("changing the ttl of a record", func(t *testing.T) { + reTTLed := shared + reTTLed.TTL = 30 + m.setStateRecords(key2, recordMapOf(reTTLed)) + requireIndexInSync(t, m) + require.Equal(t, []Record{reTTLed}, m.nameIndex[shared.Name]) + }) + + t.Run("changing the cluster nameservers", func(t *testing.T) { + m.clusterConfig.DNS = []string{"10.0.0.3"} + m.setClusterRecords() + requireIndexInSync(t, m) + require.NotContains(t, m.nameIndex, "ns2.cluster1.") + }) + + t.Run("dropping the last state key empties the index", func(t *testing.T) { + m.setStateRecords(key2, nil) + requireIndexInSync(t, m) + require.Empty(t, m.state) + for name := range m.nameIndex { + require.Contains(t, []string{"cluster1.", "ns1.cluster1."}, name, "only cluster records must be left") + } + }) +} From 29f42452ee110b69db13459d04b34c2cb12460c4 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 16:23:06 +0200 Subject: [PATCH 27/54] Frame the hb.ucast tx message once, not in every worker Each worker appended the null frame terminator to the buffer it got from the send request. That buffer is the single copy the sender shares with every per-node worker, so the appends were only safe because make([]byte, len(b)) returns a slice with no spare capacity, forcing append to allocate. A buffer allocated with room to spare would have had the workers writing their terminator into the shared array at once. Allocate the extra byte in the sender instead, where the copy is made, and let the workers write the buffer as is. The bytes on the wire are unchanged. --- daemon/hb/hbucast/hbtx.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 919dac5de..ce41e1061 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -187,17 +187,16 @@ func (t *tx) sendToNode(node, addr string, b []byte) { continue } - // Send the data with null terminator - dataWithTerminator := append(req.data, 0x00) - t.log.Tracef("sending %d bytes (+%d terminator) to %s", len(req.data), 1, addr) - if n, err := conn.Write(dataWithTerminator); err != nil { - t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(dataWithTerminator)) + // Send the data, already null terminated by the sender + t.log.Tracef("sending %d bytes to %s", len(req.data), addr) + if n, err := conn.Write(req.data); err != nil { + t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(req.data)) t.handleSendError(node, err) conn.Close() conn = nil - } else if n != len(dataWithTerminator) { - t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(dataWithTerminator)) - t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(dataWithTerminator))) + } else if n != len(req.data) { + t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(req.data)) + t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(req.data))) conn.Close() conn = nil } else { @@ -308,7 +307,10 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { continue } else { t.log.Tracef(reason) - protectedB := make([]byte, len(b)) + // The extra byte is the null frame terminator the peer rx + // scans for, left at zero by make. Framing the message here + // keeps the buffer the workers share read-only for them. + protectedB := make([]byte, len(b)+1) copy(protectedB, b) for node, addr := range t.nodes { t.sendToNode(node, addr, protectedB) From 1a6b6a45f57bffad480a78ced69c6e9e273c2d9b Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 16:30:36 +0200 Subject: [PATCH 28/54] Create the hb.ucast send queues once, in Start sendToNode() passed a freshly made channel to LoadOrStore on each call, so every send allocated a queue only to discard it, the node having had one since its first message. The lazy creation isn't needed at all: t.nodes is fixed for the lifetime of the transmitter, so Start can create the queue and the worker of each peer node up front, before the sender can reach them. sendQueues becomes a plain map, written once and only read afterwards: no sync.Map, no create on first send, no worker start race. sendToNode() is left with a queue lookup and a non blocking send, and the worker moves out of it to startSendWorker(). The node and addr fields of sendRequest go away with the same move, the worker having them from Start. --- daemon/hb/hbucast/hbtx.go | 260 ++++++++++++++++++++------------------ 1 file changed, 136 insertions(+), 124 deletions(-) diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index ce41e1061..460492980 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -35,8 +35,10 @@ type ( cmdC chan<- interface{} msgC chan<- *hbtype.Msg cancel func() - // Per-peer send queues to serialize sends to the same node - sendQueues sync.Map + // Per-peer send queues to serialize sends to the same node. Start + // creates one per configured node, before anything can use them, + // and Stop closes them once the sender is done. + sendQueues map[string]chan sendRequest // WaitGroup for send worker goroutines sendWorkers sync.WaitGroup } @@ -44,8 +46,6 @@ type ( // sendRequest holds data for a send operation type sendRequest struct { - node string - addr string data []byte // localIP is the source address the worker must dial from. It is @@ -74,12 +74,10 @@ func (t *tx) Stop() error { // before would risk a send on a closed channel. t.Wait() // Close all send queues to unblock workers - t.sendQueues.Range(func(key, value any) bool { - q := value.(chan sendRequest) - t.sendQueues.Delete(key) - close(q) - return true - }) + for node, queue := range t.sendQueues { + delete(t.sendQueues, node) + close(queue) + } t.sendWorkers.Wait() t.log.Tracef("wait done") return nil @@ -106,138 +104,143 @@ func (t *tx) streamPeerDesc(addr string) string { } // sendToNode queues a send request for a specific node -// It creates a worker goroutine for the node if one doesn't exist // // Must be called from the Start goroutine: it reads t.localIP. -func (t *tx) sendToNode(node, addr string, b []byte) { - // Get or create send queue for this node - queueI, loaded := t.sendQueues.LoadOrStore(node, make(chan sendRequest, 1)) - queue := queueI.(chan sendRequest) +func (t *tx) sendToNode(node string, b []byte) { + queue, ok := t.sendQueues[node] + if !ok { + // can't happen: Start creates a queue per configured node + t.log.Warnf("no send queue for node %s", node) + return + } - // If this is the first time we're creating the queue, start a worker - if !loaded { - t.sendWorkers.Add(1) - go func(node, addr string, q chan sendRequest) { - defer t.sendWorkers.Done() - // Worker maintains its own persistent connection - var conn net.Conn - // connLocalIP is the source address conn is bound to, to - // detect a local ip change while conn is established - var connLocalIP net.IP + // Try to send without blocking first (non-blocking send) + select { + case queue <- sendRequest{data: b, localIP: t.localIP}: + // Successfully queued + default: + // Queue is full, drop the message to avoid blocking + // This means a send is already in progress and we don't want to stack up + t.log.Tracef("send queue full for node %s, dropping message", node) + } +} - for { - select { - case <-t.ctx.Done(): - // Context cancelled, close connection and exit +// startSendWorker starts the goroutine serializing the sends to a peer +// node. It maintains its own connection, redialing when a send fails or +// the local ip changes, and exits when the transmitter context is done or +// the queue is closed. +func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { + t.sendWorkers.Add(1) + go func() { + defer t.sendWorkers.Done() + // Worker maintains its own persistent connection + var conn net.Conn + // connLocalIP is the source address conn is bound to, to + // detect a local ip change while conn is established + var connLocalIP net.IP + + for { + select { + case <-t.ctx.Done(): + // Context cancelled, close connection and exit + if conn != nil { + conn.Close() + conn = nil + } + return + case req, ok := <-q: + if !ok { + // Queue closed, exit if conn != nil { conn.Close() - conn = nil } return - case req, ok := <-q: - if !ok { - // Queue closed, exit - if conn != nil { - conn.Close() - } - return - } + } - if conn != nil && !connLocalIP.Equal(req.localIP) { - // The local ip changed since we dialed: the - // connection is bound to an address the node may - // not own anymore, redial from the new one. - t.log.Infof("local ip changed from %s to %s, reconnect to %s", connLocalIP, req.localIP, addr) - conn.Close() - conn = nil - } + if conn != nil && !connLocalIP.Equal(req.localIP) { + // The local ip changed since we dialed: the + // connection is bound to an address the node may + // not own anymore, redial from the new one. + t.log.Infof("local ip changed from %s to %s, reconnect to %s", connLocalIP, req.localIP, addr) + conn.Close() + conn = nil + } - if conn == nil { - // Create new connection with context-aware dialer - localAddr := net.TCPAddr{ - IP: req.localIP, - Port: 0, - } - dialer := &net.Dialer{ - Timeout: t.timeout, - LocalAddr: &localAddr, - } - // Use a separate context for dial that respects t.ctx. - // Cancel as soon as the dial returns: cancelling after a - // successful dial doesn't affect the connection, and this - // worker goroutine lives as long as the transmitter, so a - // deferred cancel would pile up on its stack, one per - // reconnect. - dialCtx, dialCancel := context.WithTimeout(t.ctx, t.timeout) - newConn, err := dialer.DialContext(dialCtx, "tcp", addr) - dialCancel() - if err != nil { - t.handleSendError(node, err) - continue - } - conn = newConn - connLocalIP = req.localIP + if conn == nil { + // Create new connection with context-aware dialer + localAddr := net.TCPAddr{ + IP: req.localIP, + Port: 0, } - - // Set deadline on the connection - if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { + dialer := &net.Dialer{ + Timeout: t.timeout, + LocalAddr: &localAddr, + } + // Use a separate context for dial that respects t.ctx. + // Cancel as soon as the dial returns: cancelling after a + // successful dial doesn't affect the connection, and this + // worker goroutine lives as long as the transmitter, so a + // deferred cancel would pile up on its stack, one per + // reconnect. + dialCtx, dialCancel := context.WithTimeout(t.ctx, t.timeout) + newConn, err := dialer.DialContext(dialCtx, "tcp", addr) + dialCancel() + if err != nil { t.handleSendError(node, err) - conn.Close() - conn = nil continue } + conn = newConn + connLocalIP = req.localIP + } - // Send the data, already null terminated by the sender - t.log.Tracef("sending %d bytes to %s", len(req.data), addr) - if n, err := conn.Write(req.data); err != nil { - t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(req.data)) - t.handleSendError(node, err) - conn.Close() - conn = nil - } else if n != len(req.data) { - t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(req.data)) - t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(req.data))) - conn.Close() - conn = nil - } else { - t.log.Tracef("successfully sent %d bytes to %s", len(req.data), addr) - t.clearDedupLog(node) - // Send success notification, but don't block on it - select { - case t.cmdC <- hbctrl.CmdSetPeerSuccess{ - Nodename: node, - HbID: t.id, - Success: true, - }: - case <-t.ctx.Done(): - // Context cancelled, skip notification - if conn != nil { - conn.Close() - conn = nil - } - return - } + // Set deadline on the connection + if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { + t.handleSendError(node, err) + conn.Close() + conn = nil + continue + } - // Reset deadline for next write (connection stays open) - if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { - t.log.Tracef("failed to reset deadline for %s: %v", addr, err) - // Continue with connection, it might still work + // Send the data, already null terminated by the sender + t.log.Tracef("sending %d bytes to %s", len(req.data), addr) + if n, err := conn.Write(req.data); err != nil { + t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(req.data)) + t.handleSendError(node, err) + conn.Close() + conn = nil + } else if n != len(req.data) { + t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(req.data)) + t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(req.data))) + conn.Close() + conn = nil + } else { + t.log.Tracef("successfully sent %d bytes to %s", len(req.data), addr) + t.clearDedupLog(node) + // Send success notification, but don't block on it + select { + case t.cmdC <- hbctrl.CmdSetPeerSuccess{ + Nodename: node, + HbID: t.id, + Success: true, + }: + case <-t.ctx.Done(): + // Context cancelled, skip notification + if conn != nil { + conn.Close() + conn = nil } + return + } + + // Reset deadline for next write (connection stays open) + if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { + t.log.Tracef("failed to reset deadline for %s: %v", addr, err) + // Continue with connection, it might still work } } } - }(node, addr, queue) - } - - // Try to send without blocking first (non-blocking send) - select { - case queue <- sendRequest{node: node, addr: addr, data: b, localIP: t.localIP}: - // Successfully queued - default: - // Queue is full, drop the message to avoid blocking - // This means a send is already in progress and we don't want to stack up - t.log.Tracef("send queue full for node %s, dropping message", node) - } + } + }() } // Start implements the Start function of Transmitter interface for tx @@ -250,6 +253,15 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { t.Add(1) hbaudit.EnableAudit(ctx, t.id, t.log, "hb", strings.Replace(t.id, "hb#", "hb:", 1)) + // One queue and one worker per peer node, created before the sender + // can reach them, so the map is never written again. + t.sendQueues = make(map[string]chan sendRequest, len(t.nodes)) + for node, addr := range t.nodes { + queue := make(chan sendRequest, 1) + t.sendQueues[node] = queue + t.startSendWorker(node, addr, queue) + } + go func() { defer t.Done() t.log.Infof("starting: timeout %s, interval: %s", t.timeout, t.interval) @@ -312,8 +324,8 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { // keeps the buffer the workers share read-only for them. protectedB := make([]byte, len(b)+1) copy(protectedB, b) - for node, addr := range t.nodes { - t.sendToNode(node, addr, protectedB) + for node := range t.nodes { + t.sendToNode(node, protectedB) } } } From e5e2ca18878027db3de0cf1f894920bb3ff88090 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 16:42:16 +0200 Subject: [PATCH 29/54] Tell hb.ucast rx read failures apart, and report them The read error handling compared err.Error() to "EOF" and logged both branches identically, at trace level. A peer whose frames stop decrypting, after a heartbeat secret rotation for instance, was therefore invisible at the default log level, the connection just silently dropping and being redialed. Match the errors with errors.Is and keep quiet only for the two expected ones: the peer closing the connection, and us closing it on a peer reconnect or a stop. A read deadline expiry, which means the peer went silent for 3x timeout, and any other read failure are now warnings. --- daemon/hb/hbucast/hbrx.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 85aae7d1a..65d085cc8 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" + "os" "strings" "sync" "syscall" @@ -278,10 +280,17 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { // Read will block until data arrives, connection is closed, or deadline is reached i, nodename, err := conn.ReadWithNode(buffer) if err != nil { - if err.Error() != "EOF" { - t.log.Tracef("handleLoop: read failed from %s after %d messages: %s", peerAddr, msgCount, err) - } else { + switch { + case errors.Is(err, io.EOF): + // the peer closed the connection, it will reconnect t.log.Tracef("handleLoop: EOF from %s after %d messages", peerAddr, msgCount) + case errors.Is(err, net.ErrClosed): + // we closed the connection: peer reconnect, or stop + t.log.Tracef("handleLoop: connection from %s closed after %d messages", peerAddr, msgCount) + case errors.Is(err, os.ErrDeadlineExceeded): + t.log.Warnf("handleLoop: no message from %s for %s, closing the connection", peerAddr, deadline) + default: + t.log.Warnf("handleLoop: read from %s failed after %d messages: %s", peerAddr, msgCount, err) } return } From 4f88afa355de5a0b293885890797bd23ebed37ce Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 16:46:44 +0200 Subject: [PATCH 30/54] Trim the hb.ucast per message traces A received message logged four trace lines, a sent one two, all of them saying much the same thing. Keep one per message on each side: the read one, now carrying the size, the peer node, the message kind and the count, and the write one. The rest of the traces are per connection or per error, and stay. Drop the "handleLoop:" prefix while there: the logger prefix already says which node, subsystem, direction and heartbeat the line comes from. --- daemon/hb/hbucast/hbrx.go | 26 +++++++++++--------------- daemon/hb/hbucast/hbtx.go | 3 +-- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 65d085cc8..41ce0f076 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -254,7 +254,7 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { if deadline < 10*time.Second { deadline = 10 * time.Second } - t.log.Tracef("handleLoop: starting to read messages from %s", peerAddr) + t.log.Tracef("starting to read messages from %s", peerAddr) data := msgPool.Get().([]byte) defer func() { msgPool.Put(data) }() @@ -263,14 +263,14 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { // Check context before blocking on read select { case <-t.ctx.Done(): - t.log.Tracef("handleLoop: context cancelled, stopping after %d messages from %s", msgCount, peerAddr) + t.log.Tracef("context cancelled, stopping after %d messages from %s", msgCount, peerAddr) return default: } // Set read deadline for this iteration if err := conn.SetReadDeadline(time.Now().Add(deadline)); err != nil { - t.log.Warnf("handleLoop: failed to set read deadline for %s: %v", peerAddr, err) + t.log.Warnf("failed to set read deadline for %s: %v", peerAddr, err) return } @@ -283,30 +283,28 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { switch { case errors.Is(err, io.EOF): // the peer closed the connection, it will reconnect - t.log.Tracef("handleLoop: EOF from %s after %d messages", peerAddr, msgCount) + t.log.Tracef("EOF from %s after %d messages", peerAddr, msgCount) case errors.Is(err, net.ErrClosed): // we closed the connection: peer reconnect, or stop - t.log.Tracef("handleLoop: connection from %s closed after %d messages", peerAddr, msgCount) + t.log.Tracef("connection from %s closed after %d messages", peerAddr, msgCount) case errors.Is(err, os.ErrDeadlineExceeded): - t.log.Warnf("handleLoop: no message from %s for %s, closing the connection", peerAddr, deadline) + t.log.Warnf("no message from %s for %s, closing the connection", peerAddr, deadline) default: - t.log.Warnf("handleLoop: read from %s failed after %d messages: %s", peerAddr, msgCount, err) + t.log.Warnf("read from %s failed after %d messages: %s", peerAddr, msgCount, err) } return } msgCount++ - t.log.Tracef("handleLoop: received %d bytes from node %s (msg #%d)", i, nodename, msgCount) - if i >= (msgMaxSize - 10000) { t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, peerAddr, i) } msg := hbtype.Msg{} if err := json.Unmarshal(buffer[:i], &msg); err != nil { - t.log.Warnf("handleLoop: unmarshal message failed from node %s:%s: %s", nodename, peerAddr, err) + t.log.Warnf("unmarshal message failed from node %s:%s: %s", nodename, peerAddr, err) return } - t.log.Tracef("handleLoop: successfully decoded message from node %s (kind=%s)", nodename, msg.Kind) + t.log.Tracef("read %d bytes from node %s (kind=%s, msg #%d)", i, nodename, msg.Kind, msgCount) cmdPeerSuccess := hbctrl.CmdSetPeerSuccess{ Nodename: msg.Nodename, @@ -315,17 +313,15 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { } select { case <-t.ctx.Done(): - t.log.Tracef("handleLoop: context done, stopping after %d messages", msgCount) + t.log.Tracef("context done, stopping after %d messages", msgCount) return case t.cmdC <- cmdPeerSuccess: - t.log.Tracef("handleLoop: sent peer success for node %s", msg.Nodename) } select { case <-t.ctx.Done(): - t.log.Tracef("handleLoop: context done while sending msg, stopping after %d messages", msgCount) + t.log.Tracef("context done while sending msg, stopping after %d messages", msgCount) return case t.msgC <- &msg: - t.log.Tracef("handleLoop: sent message to msgC (node=%s, kind=%s)", msg.Nodename, msg.Kind) } } } diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index 460492980..a2e7b2cf1 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -202,7 +202,6 @@ func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { } // Send the data, already null terminated by the sender - t.log.Tracef("sending %d bytes to %s", len(req.data), addr) if n, err := conn.Write(req.data); err != nil { t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(req.data)) t.handleSendError(node, err) @@ -214,7 +213,7 @@ func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { conn.Close() conn = nil } else { - t.log.Tracef("successfully sent %d bytes to %s", len(req.data), addr) + t.log.Tracef("sent %d bytes to %s", len(req.data), addr) t.clearDedupLog(node) // Send success notification, but don't block on it select { From 5a3f84eca460e8b1ce674e43517660feb430eb31 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 16:56:26 +0200 Subject: [PATCH 31/54] Make hb.ucast rx decrypt with the current heartbeat secret The receiver captured the crypto when accepting the connection, and the connections are now long lived, so it kept decrypting with the secret that was current back then. A heartbeat secret rotation then broke every established connection: the reads failed until the peer noticed its writes failing and redialed. Those failures are logged as warnings since "Tell hb.ucast rx read failures apart", which is alarming for a rotation the cluster admin asked for and which is meant to be transparent. Decrypt through hbcrypto.Loader, which resolves the crypto at each call, the way hbdisk and hbrelay refresh theirs at each tick. Nothing is dropped anymore: nmon only commits the rotation once every node has acknowledged the candidate secret, so every node then holds both keys, and the read failure warnings are back to meaning a real anomaly. --- daemon/hb/hbcrypto/main.go | 32 ++++++++++++++++++++++++++++++++ daemon/hb/hbucast/hbrx.go | 6 ++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/daemon/hb/hbcrypto/main.go b/daemon/hb/hbcrypto/main.go index 2eaf70dc7..39bd4614a 100644 --- a/daemon/hb/hbcrypto/main.go +++ b/daemon/hb/hbcrypto/main.go @@ -21,9 +21,24 @@ type ( cancel context.CancelFunc } + // Loader encrypts and decrypts with the crypto current at call time, + // instead of the one current when it was created. Users outliving a + // heartbeat secret rotation, like a hb.ucast connection, must go + // through it: a rotation is only seamless for those decrypting with + // the up to date secret, which knows both the previous and the next + // key. + Loader struct { + p *atomic.Pointer[omcrypto.T] + } + contextKey int ) +var ( + // assert Loader implements the omcrypto.EncryptDecrypter interface + _ = omcrypto.EncryptDecrypter(Loader{}) +) + const ( cryptoKey contextKey = 0 ) @@ -86,3 +101,20 @@ func CryptoFromContext(ctx context.Context) *atomic.Pointer[omcrypto.T] { } panic("context has no crypto") } + +// LoaderFromContext returns a Loader on the context crypto +func LoaderFromContext(ctx context.Context) Loader { + return Loader{p: CryptoFromContext(ctx)} +} + +func (t Loader) DecryptWithNode(data []byte) ([]byte, string, error) { + return t.p.Load().DecryptWithNode(data) +} + +func (t Loader) Decrypt(data []byte) ([]byte, error) { + return t.p.Load().Decrypt(data) +} + +func (t Loader) Encrypt(data []byte) ([]byte, error) { + return t.p.Load().Encrypt(data) +} diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 41ce0f076..fcde2ea5e 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -187,7 +187,9 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { }() t.log.Infof("listen to %s for %s", t.addr+":"+t.port, otherNodeIPL) started <- true - crypto := hbcrypto.CryptoFromContext(ctx) + // Decrypt through a loader, not through a snapshot of the crypto: + // a connection outlives heartbeat secret rotations. + crypto := hbcrypto.LoaderFromContext(ctx) for { conn, err := listener.Accept() if err != nil { @@ -209,7 +211,7 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { conn.Close() continue } - clearConn := encryptconn.New(conn, crypto.Load()) + clearConn := encryptconn.New(conn, crypto) // Check if we already have a handler for this peer and close its connection // This is done atomically in the accept loop before starting new handler if oldConnI, hasOld := t.peerConns.Load(connAddr); hasOld { From dc79c49cef7a6066659cf844993a8f29e4f130f5 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 17:05:30 +0200 Subject: [PATCH 32/54] Make hb.mcast rx decrypt with the current heartbeat secret The receiver dereferenced the crypto pointer into its own field, copying the atomic.Pointer. The copy keeps the value the pointer held at Start, and the stores the hbcrypto worker does on a heartbeat secret rotation land on the original, so the receiver decrypted with the secret current when the daemon started, forever. It survives the first rotation, the peers new key usually being the alternate key it already holds, then stops decrypting anything on the next one, until the daemon is restarted. go vet says nothing: atomic.Pointer has no noCopy. Use the hbcrypto.Loader added for hb.ucast, which resolves the pointer at each call. --- daemon/hb/hbmcast/hbrx.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/daemon/hb/hbmcast/hbrx.go b/daemon/hb/hbmcast/hbrx.go index 6be1e04ba..50aed5fc2 100644 --- a/daemon/hb/hbmcast/hbrx.go +++ b/daemon/hb/hbmcast/hbrx.go @@ -9,11 +9,9 @@ import ( "net" "strings" "sync" - "sync/atomic" "time" "github.com/opensvc/om3/v3/core/hbtype" - "github.com/opensvc/om3/v3/core/omcrypto" "github.com/opensvc/om3/v3/daemon/hb/hbaudit" "github.com/opensvc/om3/v3/daemon/hb/hbcrypto" "github.com/opensvc/om3/v3/daemon/hb/hbctrl" @@ -39,7 +37,9 @@ type ( msgC chan<- *hbtype.Msg cancel func() - crypto atomic.Pointer[omcrypto.T] + // crypto decrypts with the crypto current at call time: the + // receiver outlives heartbeat secret rotations. + crypto hbcrypto.Loader } assembly map[string]msgMap msgMap map[string]dataMap @@ -122,7 +122,7 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { } }() started <- true - t.crypto = *hbcrypto.CryptoFromContext(ctx) + t.crypto = hbcrypto.LoaderFromContext(ctx) b := make([]byte, MaxDatagramSize) for { n, src, err := listener.ReadFromUDP(b) @@ -214,9 +214,7 @@ func (t *rx) recv(src *net.UDPAddr, n int, b []byte) { } else { encMsg = chunks[1] } - crypto := t.crypto.Load() - - b, err := crypto.Decrypt(encMsg) + b, err := t.crypto.Decrypt(encMsg) if err != nil { t.log.Tracef("recv: decrypting msg from %s: %s: %s", s, hex.Dump(encMsg), err) return From 86c92e319621157790c95ecd804e65c172a0859e Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 17:54:09 +0200 Subject: [PATCH 33/54] Let the daemon tests run on a node where a daemon is alive The tests starting a daemon bound the cluster default listener port, so they failed with "listen tcp :1215: bind: address already in use" on a development node running a real daemon. daemontesthelper.SetFreeListenerPort() sets the listener.port cluster config keyword to a free port before the daemon is started. daemon.Start picks it up from the config, and republishes it to daemonenv.HTTPPort, where the test clients find it. It goes in daemontesthelper because both the daemon and the integrationtest packages can import it. TestDaemonStartupWithoutConfig has no config to set the port in, its daemon binding the listener.port keyword default, so it now skips when that port is taken, saying why. --- daemon/daemon/main_test.go | 2 ++ daemon/daemoncmd/main_test.go | 13 +++++++++++++ daemon/daemontesthelper/main.go | 29 +++++++++++++++++++++++++++++ daemon/integrationtest/main.go | 3 +++ 4 files changed, 47 insertions(+) diff --git a/daemon/daemon/main_test.go b/daemon/daemon/main_test.go index 2b895af84..f0f88354b 100644 --- a/daemon/daemon/main_test.go +++ b/daemon/daemon/main_test.go @@ -11,6 +11,7 @@ import ( "github.com/opensvc/om3/v3/core/object" "github.com/opensvc/om3/v3/core/om" "github.com/opensvc/om3/v3/daemon/daemon" + "github.com/opensvc/om3/v3/daemon/daemontesthelper" "github.com/opensvc/om3/v3/testhelper" ) @@ -25,6 +26,7 @@ func setup(t *testing.T) testhelper.Env { env.InstallFile("../../testdata/ca-cluster1.conf", "etc/namespaces/system/sec/ca.conf") env.InstallFile("../../testdata/cert-cluster1.conf", "etc/namespaces/system/sec/cert.conf") env.InstallFile("../../testdata/hb.conf", "etc/namespaces/system/sec/hb.conf") + daemontesthelper.SetFreeListenerPort(t) // daemondata.Start needs initial cluster.ConfigData.Set _, err := object.SetClusterConfig() require.NoError(t, err) diff --git a/daemon/daemoncmd/main_test.go b/daemon/daemoncmd/main_test.go index f2e70c43a..176fa1c53 100644 --- a/daemon/daemoncmd/main_test.go +++ b/daemon/daemoncmd/main_test.go @@ -25,6 +25,13 @@ import ( "github.com/opensvc/om3/v3/util/plog" ) +// defaultHTTPPort is the port a daemon started with no cluster config +// listens on. Captured here because daemon.Start overwrites +// daemonenv.HTTPPort with the port of the config it starts with, while +// the listener.port keyword default, which the configless daemon uses, +// keeps the value this package var holds. +var defaultHTTPPort = daemonenv.HTTPPort + func newClient(serverUrl string) (*client.T, error) { return client.New(client.WithURL(serverUrl), client.WithPassword(cluster.ConfigData.Get().Secret())) //return client.New(client.WithURL(serverUrl), client.WithInsecureSkipVerify(true)) @@ -250,5 +257,11 @@ func TestDaemonStartupWithoutConfig(t *testing.T) { if runtime.GOOS != "darwin" && os.Getuid() != 0 { t.Skip("skipped for non root user") } + // This one starts a daemon with no cluster config, so it can't be told + // to listen elsewhere: the port comes from the listener.port keyword + // default. A daemon alive on this node holds it. + if err := testhelper.TCPPortAvailable(fmt.Sprint(defaultHTTPPort)); err != nil { + t.Skipf("skipped: a daemon started without config needs port %d: %s", defaultHTTPPort, err) + } runTestDaemonStartup(t, false) } diff --git a/daemon/daemontesthelper/main.go b/daemon/daemontesthelper/main.go index 7407f6f11..58ab860ec 100644 --- a/daemon/daemontesthelper/main.go +++ b/daemon/daemontesthelper/main.go @@ -3,6 +3,8 @@ package daemontesthelper import ( "context" + "fmt" + "net" "os" "testing" "time" @@ -15,6 +17,7 @@ import ( "github.com/opensvc/om3/v3/core/cluster" "github.com/opensvc/om3/v3/core/hbsecobject" "github.com/opensvc/om3/v3/core/instance" + "github.com/opensvc/om3/v3/core/keyop" "github.com/opensvc/om3/v3/core/node" "github.com/opensvc/om3/v3/core/object" "github.com/opensvc/om3/v3/core/rawconfig" @@ -100,6 +103,32 @@ func Setup(t *testing.T, env *testhelper.Env) *D { } } +// SetFreeListenerPort points the listener.port cluster config keyword to a +// free port, so a daemon started by a test doesn't try to bind the cluster +// default one, which a real daemon may hold on the test node. +// +// Call it after the cluster config is installed, and before the daemon is +// started: daemon.Start reads the port from the config and republishes it +// to daemonenv.HTTPPort, where the clients find it. +func SetFreeListenerPort(t *testing.T) { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + require.NoError(t, listener.Close()) + + o, err := object.NewCluster() + require.NoError(t, err) + require.NoError(t, o.Config().Set(keyop.ParseList(fmt.Sprintf("listener.port=%d", port))...)) + + // the tests of a binary share daemonenv.HTTPPort + previousPort := daemonenv.HTTPPort + t.Cleanup(func() { daemonenv.HTTPPort = previousPort }) + + t.Logf("listener port set to %d", port) +} + func initEnv(t *testing.T) *testhelper.Env { env := testhelper.Setup(t) t.Logf("Starting daemon with OSVC_ROOT_PATH=%s", env.Root) diff --git a/daemon/integrationtest/main.go b/daemon/integrationtest/main.go index 4e624e389..516df9c9b 100644 --- a/daemon/integrationtest/main.go +++ b/daemon/integrationtest/main.go @@ -16,6 +16,7 @@ import ( "github.com/opensvc/om3/v3/core/rawconfig" "github.com/opensvc/om3/v3/daemon/daemon" "github.com/opensvc/om3/v3/daemon/daemonenv" + "github.com/opensvc/om3/v3/daemon/daemontesthelper" "github.com/opensvc/om3/v3/testhelper" "github.com/opensvc/om3/v3/util/hostname" ) @@ -40,6 +41,8 @@ func Setup(t *testing.T) (testhelper.Env, func()) { env.InstallFile("./testdata/cert-cluster1.conf", "etc/namespaces/system/sec/cert.conf") env.InstallFile("./testdata/hb.conf", "etc/namespaces/system/sec/hb.conf") + daemontesthelper.SetFreeListenerPort(t) + // daemondata.Start needs initial cluster.ConfigData.Set _, err := object.SetClusterConfig() require.NoError(t, err) From 5dc07e23db40dcc3436974c7fca50c43249eb261 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 17:59:48 +0200 Subject: [PATCH 34/54] Test TCPPortAvailable on a free port, not on the daemon one Test_TcpPortAvailable asserted 1215 was available, which it isn't on a node running a daemon. The test only needs a port nobody listens on: ask the kernel for one. --- testhelper/main_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testhelper/main_test.go b/testhelper/main_test.go index 07e29d7b2..521bfa563 100644 --- a/testhelper/main_test.go +++ b/testhelper/main_test.go @@ -2,13 +2,20 @@ package testhelper import ( "net" + "strconv" "testing" "github.com/stretchr/testify/require" ) func Test_TcpPortAvailable(t *testing.T) { - port := "1215" + // Any free port will do. Don't pick a well known one: the node may run + // a real daemon holding it. + freeLn, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := strconv.Itoa(freeLn.Addr().(*net.TCPAddr).Port) + require.NoError(t, freeLn.Close()) + require.NoErrorf(t, TCPPortAvailable(port), "port %s should be available before test", port) Trace(t) if t.Failed() { From b05e6b4c07f42842d84d49fda93f40e3b1eea2dc Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 18:33:35 +0200 Subject: [PATCH 35/54] Re-resolve the hb.ucast rx peers when an unknown address connects The receiver resolved the peer node names once, at start, and rejected every connection coming from an address absent from that snapshot. The transmitter, on the other hand, refreshes its local address every 30s and rebinds its connections to it. So a node whose address changed kept beating, from an address its peers refused, until their receiver was restarted. Both sides read the same source of truth: the transmitter dials from the address the local node name resolves to (defaultLocalIP), and the receiver allows the addresses that same name resolves to. A connection from an unknown address is therefore the signal that a peer may have moved, and resolving the names again is enough to confirm it. Do it in the accept loop, on the rejection path: - the accept loop is the only owner of the allow list, so the refresh needs no locking, unlike a ticker in a goroutine of its own - a known address never triggers a lookup, so the steady state is unchanged - one refresh per timeout at most, so a stranger hammering the port can't turn into a lookup storm - a node whose lookup fails keeps its known addresses, so a resolver hiccup doesn't empty the allow list The listener closing goroutine now gets its own copy of the address list, which the accept loop writes to. --- daemon/hb/hbucast/hbrx.go | 98 +++++++++++++++++++++++++++------- daemon/hb/hbucast/hbrx_test.go | 59 ++++++++++++++++++++ 2 files changed, 139 insertions(+), 18 deletions(-) create mode 100644 daemon/hb/hbucast/hbrx_test.go diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index fcde2ea5e..469b4d8c2 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -8,6 +8,7 @@ import ( "io" "net" "os" + "slices" "strings" "sync" "syscall" @@ -149,9 +150,6 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { t.Add(1) go func() { defer t.Done() - otherNodeIPM := make(map[string]struct{}) - otherNodeIPL := make([]string, 0) - resolver := net.Resolver{} for node, addr := range t.nodes { cmdC <- hbctrl.CmdAddWatcher{ @@ -161,30 +159,33 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { Timeout: t.timeout, Desc: t.streamPeerDesc(addr), } - addr, _, _ := strings.Cut(addr, ":") - addrs, err := resolver.LookupHost(ctx, addr) - if err != nil { - continue - } - for _, addr := range addrs { - t.log.Infof("add expected %s address: %s", node, addr) - otherNodeIPM[addr] = struct{}{} - otherNodeIPL = append(otherNodeIPL, addr) + } + + // peerIPs, and the set and list derived from it, are owned by this + // accept loop, which refreshes them when an unknown address shows up. + peerIPs := t.resolvePeerIPs(ctx, nil) + resolvedAt := time.Now() + otherNodeIPM, otherNodeIPL := peerIPSet(peerIPs) + logExpected := func() { + for node, addrs := range peerIPs { + t.log.Infof("add expected %s address: %s", node, addrs) } } + logExpected() + var wg sync.WaitGroup wg.Add(1) - go func() { + go func(peerIPL []string) { defer wg.Done() select { case <-ctx.Done(): - t.log.Infof("closing listener %s for %s", t.addr+":"+t.port, otherNodeIPL) + t.log.Infof("closing listener %s for %s", t.addr+":"+t.port, peerIPL) _ = listener.Close() time.Sleep(100 * time.Millisecond) t.cancel() return } - }() + }(otherNodeIPL) t.log.Infof("listen to %s for %s", t.addr+":"+t.port, otherNodeIPL) started <- true // Decrypt through a loader, not through a snapshot of the crypto: @@ -207,9 +208,25 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { continue } if _, ok := otherNodeIPM[connAddr]; !ok { - t.log.Warnf("unexpected connection from %s", connAddr) - conn.Close() - continue + // A connection from an unknown address is the signal that a + // peer may have moved: the tx dials from the address its own + // name resolves to, so the name we resolved at startup now + // points elsewhere. Resolve again, rate limited so that a + // stranger hammering the port can't turn into a lookup storm. + if time.Since(resolvedAt) > t.timeout { + resolvedAt = time.Now() + peerIPs = t.resolvePeerIPs(ctx, peerIPs) + if set, list := peerIPSet(peerIPs); !slices.Equal(list, otherNodeIPL) { + otherNodeIPM, otherNodeIPL = set, list + logExpected() + } + } + if _, ok := otherNodeIPM[connAddr]; !ok { + t.log.Warnf("unexpected connection from %s", connAddr) + conn.Close() + continue + } + t.log.Infof("accept connection from %s, a peer address changed", connAddr) } clearConn := encryptconn.New(conn, crypto) // Check if we already have a handler for this peer and close its connection @@ -248,6 +265,51 @@ func (t *rx) Start(cmdC chan<- interface{}, msgC chan<- *hbtype.Msg) error { return nil } +// resolvePeerIPs returns the addresses of each peer node, indexed by node. +// +// A node whose lookup fails keeps the addresses it has in previous, so that +// a transient resolver failure doesn't empty the allow list the accept loop +// checks the connections against. +func (t *rx) resolvePeerIPs(ctx context.Context, previous map[string][]string) map[string][]string { + resolver := net.Resolver{} + peerIPs := make(map[string][]string, len(t.nodes)) + for node, addr := range t.nodes { + addr, _, _ := strings.Cut(addr, ":") + addrs, err := resolver.LookupHost(ctx, addr) + if err != nil { + if kept, ok := previous[node]; ok { + t.log.Debugf("lookup %s: %s: keep the known addresses %s", node, err, kept) + peerIPs[node] = kept + } else { + t.log.Debugf("lookup %s: %s", node, err) + } + continue + } + slices.Sort(addrs) + peerIPs[node] = addrs + } + return peerIPs +} + +// peerIPSet returns the addresses of peerIPs as a set, to check the source +// address of a connection against, and as a sorted list, to log them and to +// tell two resolutions apart. +func peerIPSet(peerIPs map[string][]string) (map[string]struct{}, []string) { + set := make(map[string]struct{}) + list := make([]string, 0) + for _, addrs := range peerIPs { + for _, addr := range addrs { + if _, ok := set[addr]; ok { + continue + } + set[addr] = struct{}{} + list = append(list, addr) + } + } + slices.Sort(list) + return set, list +} + func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { // Set a generous read deadline to prevent idle connections from blocking indefinitely. // This is long enough to cover the heartbeat interval (default 5s) with some margin. diff --git a/daemon/hb/hbucast/hbrx_test.go b/daemon/hb/hbucast/hbrx_test.go new file mode 100644 index 000000000..6ea19476b --- /dev/null +++ b/daemon/hb/hbucast/hbrx_test.go @@ -0,0 +1,59 @@ +package hbucast + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opensvc/om3/v3/util/plog" +) + +func TestPeerIPSet(t *testing.T) { + set, list := peerIPSet(map[string][]string{ + "node2": {"10.0.0.2", "10.0.1.2"}, + // node3 has a floating address it shares with node2 + "node3": {"10.0.1.2", "10.0.0.3"}, + }) + require.Equal(t, []string{"10.0.0.2", "10.0.0.3", "10.0.1.2"}, list, + "the list is sorted and holds each address once") + for _, addr := range list { + require.Contains(t, set, addr) + } + require.Len(t, set, len(list)) + + set, list = peerIPSet(nil) + require.Empty(t, set) + require.Empty(t, list) +} + +func TestResolvePeerIPs(t *testing.T) { + ctx := context.Background() + // a name reserved by RFC 6761 for the purpose, it must not resolve + unresolvable := "node2.invalid" + if _, err := net.DefaultResolver.LookupHost(ctx, unresolvable); err == nil { + t.Skipf("%s resolves on this node, the lookup failure path can't be tested", unresolvable) + } + + newRx := func(nodes map[string]string) *rx { + return &rx{nodes: nodes, log: plog.NewDefaultLogger()} + } + + t.Run("an address is resolved to itself", func(t *testing.T) { + r := newRx(map[string]string{"node2": "10.0.0.2:10000"}) + require.Equal(t, map[string][]string{"node2": {"10.0.0.2"}}, r.resolvePeerIPs(ctx, nil)) + }) + + t.Run("a node that doesn't resolve has no address", func(t *testing.T) { + r := newRx(map[string]string{"node2": unresolvable + ":10000"}) + require.Empty(t, r.resolvePeerIPs(ctx, nil)) + }) + + t.Run("a node that stops resolving keeps its known addresses", func(t *testing.T) { + r := newRx(map[string]string{"node2": unresolvable + ":10000"}) + previous := map[string][]string{"node2": {"10.0.0.2"}} + require.Equal(t, previous, r.resolvePeerIPs(ctx, previous), + "a transient resolver failure must not empty the allow list") + }) +} From 0948711e106382aacca047011e406da36b8eaa25 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 18:55:51 +0200 Subject: [PATCH 36/54] Don't hold a max sized read buffer per hb.ucast rx connection Each handler took a msgMaxSize buffer from a pool, and held it for the life of the connection, because ReadWithNode copies the message into a caller buffer that must be large enough for the biggest message to come. So a peer holding an idle connection held 10MB, before sending anything, and before any message authentication: the accept loop only checks the source address against the peer address list. MessageWithNode returns the decrypted message instead of copying it into a caller buffer. The slice is sized for the message read, so an idle connection holds nothing but the scanner buffer, which starts at the usual message size, and the copy is one less. The 10MB stays reachable, the scanner growing to the max message size for a peer that really sends one, which is what the max is for. It now takes sending it. --- daemon/encryptconn/main.go | 25 ++++++++++--- daemon/encryptconn/main_test.go | 62 +++++++++++++++++++++++++++++++++ daemon/hb/hbucast/hbrx.go | 28 +++++---------- 3 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 daemon/encryptconn/main_test.go diff --git a/daemon/encryptconn/main.go b/daemon/encryptconn/main.go index c30042c7e..fe084b57e 100644 --- a/daemon/encryptconn/main.go +++ b/daemon/encryptconn/main.go @@ -29,6 +29,7 @@ type ( ConnNoder interface { net.Conn ReadWithNode(b []byte) (n int, nodename string, err error) + MessageWithNode() (b []byte, nodename string, err error) } ) @@ -79,18 +80,32 @@ func (t *T) Read(b []byte) (n int, err error) { // ReadWithNode implement ConnNoder interface for T // // read and decrypt data read from t.Conn +// +// Prefer MessageWithNode when the message size is not known in advance: b +// has to be as large as the largest message to expect, where the slice +// MessageWithNode returns is as large as the message actually read. func (t *T) ReadWithNode(b []byte) (n int, nodename string, err error) { - var encBytes, clearBytes []byte - if encBytes, err = t.getMessage(); err != nil { - return - } - if clearBytes, nodename, err = t.encryptDecrypter.DecryptWithNode(encBytes); err != nil { + var clearBytes []byte + if clearBytes, nodename, err = t.MessageWithNode(); err != nil { return } n = copy(b, clearBytes) return } +// MessageWithNode implement ConnNoder interface for T +// +// read and decrypt the next message from t.Conn, and return it with the +// nodename of its encrypter. The returned slice is sized for the message +// and belongs to the caller: a connection reading nothing retains nothing. +func (t *T) MessageWithNode() (b []byte, nodename string, err error) { + var encBytes []byte + if encBytes, err = t.getMessage(); err != nil { + return + } + return t.encryptDecrypter.DecryptWithNode(encBytes) +} + // SrcNode returns the encrypter nodename func (t *T) SrcNode() string { return t.srcNode diff --git a/daemon/encryptconn/main_test.go b/daemon/encryptconn/main_test.go new file mode 100644 index 000000000..b859fcb98 --- /dev/null +++ b/daemon/encryptconn/main_test.go @@ -0,0 +1,62 @@ +package encryptconn + +import ( + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +// clearCrypto is a pass through encryptDecrypter, to test the framing +// without the encryption noise. +type clearCrypto struct{} + +func (clearCrypto) Encrypt(b []byte) ([]byte, error) { return b, nil } + +func (clearCrypto) DecryptWithNode(b []byte) ([]byte, string, error) { return b, "node2", nil } + +// newTestConn returns a *T reading from the returned writer. +func newTestConn(t *testing.T) (*T, net.Conn) { + t.Helper() + peer, local := net.Pipe() + t.Cleanup(func() { + _ = peer.Close() + _ = local.Close() + }) + return New(local, clearCrypto{}), peer +} + +func TestMessageWithNode(t *testing.T) { + conn, peer := newTestConn(t) + + go func() { + // two frames in a single write, as tcp coalescing does + _, _ = peer.Write([]byte("first\x00second\x00")) + }() + + b, nodename, err := conn.MessageWithNode() + require.NoError(t, err) + require.Equal(t, "node2", nodename) + require.Equal(t, "first", string(b)) + require.Equal(t, len(b), cap(b), + "the returned message must be sized for the message, so that a connection retains no more than what it read") + + b, _, err = conn.MessageWithNode() + require.NoError(t, err) + require.Equal(t, "second", string(b), "the frames buffered by the scanner must not be lost") + require.Equal(t, len(b), cap(b)) +} + +func TestReadWithNode(t *testing.T) { + conn, peer := newTestConn(t) + + go func() { + _, _ = peer.Write([]byte("first\x00")) + }() + + b := make([]byte, 512) + n, nodename, err := conn.ReadWithNode(b) + require.NoError(t, err) + require.Equal(t, "node2", nodename) + require.Equal(t, "first", string(b[:n])) +} diff --git a/daemon/hb/hbucast/hbrx.go b/daemon/hb/hbucast/hbrx.go index 469b4d8c2..e081f52ac 100644 --- a/daemon/hb/hbucast/hbrx.go +++ b/daemon/hb/hbucast/hbrx.go @@ -48,15 +48,6 @@ type ( var ( msgMaxSize = 10000000 // max kind=full msg size - - // Create a new sync.Pool to manage the byte buffers. Used to reduce memory usage - // during handling the messages. - msgPool = sync.Pool{ - New: func() interface{} { - // This creates a new byte slice of the specified size. - return make([]byte, msgMaxSize) - }, - } ) // ID implements the ID function of the Receiver interface for rx @@ -319,8 +310,6 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { deadline = 10 * time.Second } t.log.Tracef("starting to read messages from %s", peerAddr) - data := msgPool.Get().([]byte) - defer func() { msgPool.Put(data) }() msgCount := 0 for { @@ -338,11 +327,10 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { return } - // Reset buffer for each message - buffer := data[:cap(data)] - - // Read will block until data arrives, connection is closed, or deadline is reached - i, nodename, err := conn.ReadWithNode(buffer) + // Read will block until data arrives, connection is closed, or + // deadline is reached. The returned message is sized for the frame + // read, so an idle connection retains nothing. + b, nodename, err := conn.MessageWithNode() if err != nil { switch { case errors.Is(err, io.EOF): @@ -360,15 +348,15 @@ func (t *rx) handleLoop(conn encryptconn.ConnNoder, peerAddr string) { } msgCount++ - if i >= (msgMaxSize - 10000) { - t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, peerAddr, i) + if len(b) >= (msgMaxSize - 10000) { + t.log.Warnf("read huge message from node %s:%s msg size: %d", nodename, peerAddr, len(b)) } msg := hbtype.Msg{} - if err := json.Unmarshal(buffer[:i], &msg); err != nil { + if err := json.Unmarshal(b, &msg); err != nil { t.log.Warnf("unmarshal message failed from node %s:%s: %s", nodename, peerAddr, err) return } - t.log.Tracef("read %d bytes from node %s (kind=%s, msg #%d)", i, nodename, msg.Kind, msgCount) + t.log.Tracef("read %d bytes from node %s (kind=%s, msg #%d)", len(b), nodename, msg.Kind, msgCount) cmdPeerSuccess := hbctrl.CmdSetPeerSuccess{ Nodename: msg.Nodename, From 1a00b3e4900d6cf801d05d6a3bd3d82815c582b2 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 27 Aug 2026 19:07:38 +0200 Subject: [PATCH 37/54] Close the hb.ucast tx connections on stop, not just the queues Stop closed the send queues and waited for the workers, but not their connections, which the conn pool removal had taken away. A worker parked in a write to a peer that stopped reading is only released by its own deadline, so stopping or reconfiguring the heartbeat took a timeout, 15s with the defaults, instead of being immediate. The connection each worker dials is now published to its sendWorker, behind a mutex, so that Stop can close it and end the write. The worker keeps owning the dial and the send: the mutex guards the hand off to Stop only, and closing a connection under a blocked write is the way to interrupt it. handleSendError returns early once the context is done, so the write Stop just interrupted doesn't open a send error period on the way out. The dial was already interrupted by the context, and the send success notification already gave up on it, contrary to what the report says. --- daemon/hb/hbucast/hbtx.go | 127 +++++++++++++++++++++------------ daemon/hb/hbucast/hbtx_test.go | 59 +++++++++++++++ 2 files changed, 139 insertions(+), 47 deletions(-) create mode 100644 daemon/hb/hbucast/hbtx_test.go diff --git a/daemon/hb/hbucast/hbtx.go b/daemon/hb/hbucast/hbtx.go index a2e7b2cf1..ab2a712e6 100644 --- a/daemon/hb/hbucast/hbtx.go +++ b/daemon/hb/hbucast/hbtx.go @@ -35,12 +35,12 @@ type ( cmdC chan<- interface{} msgC chan<- *hbtype.Msg cancel func() - // Per-peer send queues to serialize sends to the same node. Start - // creates one per configured node, before anything can use them, - // and Stop closes them once the sender is done. - sendQueues map[string]chan sendRequest + // Per-peer send workers, to serialize the sends to the same node. + // Start creates one per configured node, before anything can use + // them, and Stop closes them once the sender is done. + sendWorkers map[string]*sendWorker // WaitGroup for send worker goroutines - sendWorkers sync.WaitGroup + sendWorkersWG sync.WaitGroup } ) @@ -54,6 +54,42 @@ type sendRequest struct { localIP net.IP } +// sendWorker serializes the sends to one peer node +type sendWorker struct { + queue chan sendRequest + + // mu protects conn, which the worker goroutine owns, and Stop closes + // to interrupt a write to a peer that stopped reading. Its deadline + // would otherwise hold the shutdown for a whole timeout. + mu sync.Mutex + conn net.Conn +} + +// getConn returns the worker connection, nil when it has to dial one +func (w *sendWorker) getConn() net.Conn { + w.mu.Lock() + defer w.mu.Unlock() + return w.conn +} + +// setConn publishes the connection the worker just dialed +func (w *sendWorker) setConn(conn net.Conn) { + w.mu.Lock() + defer w.mu.Unlock() + w.conn = conn +} + +// closeConn closes the worker connection, if it has one. The worker calls +// it when a send fails, and Stop to interrupt a blocked send. +func (w *sendWorker) closeConn() { + w.mu.Lock() + defer w.mu.Unlock() + if w.conn != nil { + _ = w.conn.Close() + w.conn = nil + } +} + // ID implements the ID function of Transmitter interface for tx func (t *tx) ID() string { return t.id @@ -73,12 +109,15 @@ func (t *tx) Stop() error { // so the send queues have no writer left once it is done. Closing them // before would risk a send on a closed channel. t.Wait() - // Close all send queues to unblock workers - for node, queue := range t.sendQueues { - delete(t.sendQueues, node) - close(queue) + // Close the queues to unblock the workers, and their connections: a + // worker can be parked in a write to a peer that stopped reading, and + // only its own deadline, a timeout away, would end it. + for node, w := range t.sendWorkers { + delete(t.sendWorkers, node) + close(w.queue) + w.closeConn() } - t.sendWorkers.Wait() + t.sendWorkersWG.Wait() t.log.Tracef("wait done") return nil } @@ -107,16 +146,16 @@ func (t *tx) streamPeerDesc(addr string) string { // // Must be called from the Start goroutine: it reads t.localIP. func (t *tx) sendToNode(node string, b []byte) { - queue, ok := t.sendQueues[node] + w, ok := t.sendWorkers[node] if !ok { - // can't happen: Start creates a queue per configured node - t.log.Warnf("no send queue for node %s", node) + // can't happen: Start creates a worker per configured node + t.log.Warnf("no send worker for node %s", node) return } // Try to send without blocking first (non-blocking send) select { - case queue <- sendRequest{data: b, localIP: t.localIP}: + case w.queue <- sendRequest{data: b, localIP: t.localIP}: // Successfully queued default: // Queue is full, drop the message to avoid blocking @@ -129,40 +168,36 @@ func (t *tx) sendToNode(node string, b []byte) { // node. It maintains its own connection, redialing when a send fails or // the local ip changes, and exits when the transmitter context is done or // the queue is closed. -func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { - t.sendWorkers.Add(1) +func (t *tx) startSendWorker(node, addr string, w *sendWorker) { + t.sendWorkersWG.Add(1) go func() { - defer t.sendWorkers.Done() - // Worker maintains its own persistent connection - var conn net.Conn - // connLocalIP is the source address conn is bound to, to - // detect a local ip change while conn is established + defer t.sendWorkersWG.Done() + // The worker connection is closed on every exit path, and by Stop + // when it has to interrupt a send. + defer w.closeConn() + // connLocalIP is the source address the connection is bound to, to + // detect a local ip change while it is established var connLocalIP net.IP for { select { case <-t.ctx.Done(): - // Context cancelled, close connection and exit - if conn != nil { - conn.Close() - conn = nil - } + // Context cancelled, exit return - case req, ok := <-q: + case req, ok := <-w.queue: if !ok { // Queue closed, exit - if conn != nil { - conn.Close() - } return } + conn := w.getConn() + if conn != nil && !connLocalIP.Equal(req.localIP) { // The local ip changed since we dialed: the // connection is bound to an address the node may // not own anymore, redial from the new one. t.log.Infof("local ip changed from %s to %s, reconnect to %s", connLocalIP, req.localIP, addr) - conn.Close() + w.closeConn() conn = nil } @@ -191,13 +226,13 @@ func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { } conn = newConn connLocalIP = req.localIP + w.setConn(conn) } // Set deadline on the connection if err := conn.SetDeadline(time.Now().Add(t.timeout)); err != nil { t.handleSendError(node, err) - conn.Close() - conn = nil + w.closeConn() continue } @@ -205,13 +240,11 @@ func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { if n, err := conn.Write(req.data); err != nil { t.log.Tracef("write failed to %s: %v (wrote %d/%d bytes)", addr, err, n, len(req.data)) t.handleSendError(node, err) - conn.Close() - conn = nil + w.closeConn() } else if n != len(req.data) { t.log.Tracef("short write to %s: %d/%d bytes", addr, n, len(req.data)) t.handleSendError(node, fmt.Errorf("short write: %d/%d", n, len(req.data))) - conn.Close() - conn = nil + w.closeConn() } else { t.log.Tracef("sent %d bytes to %s", len(req.data), addr) t.clearDedupLog(node) @@ -224,10 +257,6 @@ func (t *tx) startSendWorker(node, addr string, q chan sendRequest) { }: case <-t.ctx.Done(): // Context cancelled, skip notification - if conn != nil { - conn.Close() - conn = nil - } return } @@ -252,13 +281,13 @@ func (t *tx) Start(cmdC chan<- interface{}, msgC <-chan []byte) error { t.Add(1) hbaudit.EnableAudit(ctx, t.id, t.log, "hb", strings.Replace(t.id, "hb#", "hb:", 1)) - // One queue and one worker per peer node, created before the sender - // can reach them, so the map is never written again. - t.sendQueues = make(map[string]chan sendRequest, len(t.nodes)) + // One worker per peer node, created before the sender can reach them, + // so the map is never written again. + t.sendWorkers = make(map[string]*sendWorker, len(t.nodes)) for node, addr := range t.nodes { - queue := make(chan sendRequest, 1) - t.sendQueues[node] = queue - t.startSendWorker(node, addr, queue) + w := &sendWorker{queue: make(chan sendRequest, 1)} + t.sendWorkers[node] = w + t.startSendWorker(node, addr, w) } go func() { @@ -352,6 +381,10 @@ func (t *tx) defaultLocalIP() (net.IP, error) { // handleSendError handles send errors with deduplication logging func (t *tx) handleSendError(node string, err error) { + if t.ctx.Err() != nil { + // stopping: the error is the dial or the send Stop just interrupted + return + } newErr := err.Error() if lastErr, ok := t.lastNodeErr.Load(node); ok { if lastErr == newErr { diff --git a/daemon/hb/hbucast/hbtx_test.go b/daemon/hb/hbucast/hbtx_test.go new file mode 100644 index 000000000..908a1e6aa --- /dev/null +++ b/daemon/hb/hbucast/hbtx_test.go @@ -0,0 +1,59 @@ +package hbucast + +import ( + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestSendWorkerCloseConn verifies the mechanism Stop relies on to not wait +// a timeout for a worker parked in a send to a peer that stopped reading. +func TestSendWorkerCloseConn(t *testing.T) { + t.Run("closing interrupts a blocked write", func(t *testing.T) { + peer, local := net.Pipe() + defer peer.Close() + + w := &sendWorker{queue: make(chan sendRequest, 1)} + w.setConn(local) + + // net.Pipe is synchronous: this write blocks until the peer reads, + // as a write to a peer whose receive window is full does. + errC := make(chan error, 1) + go func() { + _, err := w.getConn().Write([]byte("a message\x00")) + errC <- err + }() + + select { + case err := <-errC: + t.Fatalf("the write returned before the peer read: %v", err) + case <-time.After(100 * time.Millisecond): + } + + w.closeConn() + + select { + case err := <-errC: + require.Error(t, err, "the interrupted write must fail") + case <-time.After(time.Second): + t.Fatal("closeConn did not interrupt the blocked write") + } + require.Nil(t, w.getConn(), "a closed connection must not be handed out again") + }) + + t.Run("closing twice is harmless", func(t *testing.T) { + peer, local := net.Pipe() + defer peer.Close() + + w := &sendWorker{queue: make(chan sendRequest, 1)} + // the worker closing on its way out and Stop closing to interrupt it + // can both happen + w.closeConn() + w.setConn(local) + w.closeConn() + w.closeConn() + require.Nil(t, w.getConn()) + }) +} From c42f223a655588d898d226a836712ef2d990817b Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 08:28:58 +0200 Subject: [PATCH 38/54] Fix the tui lockup on page down in the heartbeats and relay views tview's Table page-up/page-down handlers snapshot the current selection before clamping it and use it as the sentinel terminating their cell scan. When a table has selection enabled but not a single selectable cell, Table.Draw() has already pushed the selection one row past the last one looking for a selectable cell, so that sentinel is never reached and the scan loops forever, wedging the whole application. createTable() enabled selection unconditionally, while the heartbeats view declares no selectable column (and the relay view declares none at all). Both were a page down away from a hang, whatever their row count. A view declaring no selectable column is a plain scrollable table: don't enable selection on it. Page up and page down then move the row offset, which is the wanted behaviour there. As those views have no selection to restore from t.position, carry the scroll offset over the periodic table rebuild, so a heartbeat status change does not snap the reader back to the first row. --- core/tui/createtable_hang_test.go | 61 +++++++++++++++++++++++++++++++ core/tui/main.go | 26 ++++++++++--- 2 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 core/tui/createtable_hang_test.go diff --git a/core/tui/createtable_hang_test.go b/core/tui/createtable_hang_test.go new file mode 100644 index 000000000..bc484826e --- /dev/null +++ b/core/tui/createtable_hang_test.go @@ -0,0 +1,61 @@ +package tui + +import ( + "fmt" + "testing" + "time" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +// A table declaring no selectable column must not lock the event loop up on +// page-down / page-up. +func TestCreateTableNoSelectableColumnPaging(t *testing.T) { + for _, key := range []tcell.Key{tcell.KeyPgDn, tcell.KeyPgUp} { + screen := tcell.NewSimulationScreen("UTF-8") + if err := screen.Init(); err != nil { + t.Fatal(err) + } + screen.SetSize(120, 10) + + app := NewApp(nil) + app.app = tview.NewApplication().SetScreen(screen) + app.initHeadTextView() + app.initErrsTextView() + app.flex = tview.NewFlex().SetDirection(tview.FlexRow) + app.flex.AddItem(app.head, 1, 0, false) + app.app.SetRoot(app.flex, true) + + titles := []string{"RUNNING", "BEATING", "ID", "NODE", "PEER", "TYPE", "DESC", "CHANGED_AT"} + elements := make([][]string, 0, 40) + for r := 0; r < 40; r++ { + row := make([]string, len(titles)) + for c := range titles { + row[c] = fmt.Sprintf("r%dc%d", r, c) + } + elements = append(elements, row) + } + app.createTable(CreateTableOptions{ + title: "heartbeats", + titles: titles, + elementsList: elements, + selectableColumns: []int{}, + }) + + done := make(chan struct{}) + go func() { + defer close(done) + _ = app.app.Run() + }() + time.Sleep(200 * time.Millisecond) + screen.InjectKey(key, 0, tcell.ModNone) + time.Sleep(200 * time.Millisecond) + app.app.Stop() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatalf("event loop locked up on key %v", tcell.KeyNames[key]) + } + } +} diff --git a/core/tui/main.go b/core/tui/main.go index 9c8f7554e..bdedd33ef 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -2462,8 +2462,15 @@ func (t *App) createTable(creator CreateTableOptions) { if t.focused { return } + // tview's table page-up/page-down handlers spin forever on a table that has + // selection enabled but not a single selectable cell: they use the current + // selection as the sentinel of their cell scan, and Table.Draw() has already + // pushed that selection past the last row looking for a selectable cell. + // Views declaring no selectable column are plain scrollable tables. + isSelectable := len(creator.selectableColumns) > 0 + v := tview.NewTable() - v.SetSelectable(true, true) + v.SetSelectable(isSelectable, isSelectable) v.SetTitle(creator.title) update := func() { @@ -2495,12 +2502,14 @@ func (t *App) createTable(creator CreateTableOptions) { for i, elements := range creator.elementsList { row := i + 1 for j, element := range elements { - selectable := creator.selectableColumns != nil && slices.Contains(creator.selectableColumns, j) + selectable := slices.Contains(creator.selectableColumns, j) v.SetCell(row, j, tview.NewTableCell(element).SetSelectable(selectable)) } } - v.Select(t.position.row, t.position.col) + if isSelectable { + v.Select(t.position.row, t.position.col) + } tablesDiffer := func(a, b *tview.Table) bool { if a.GetColumnCount() != b.GetColumnCount() || a.GetRowCount() != b.GetRowCount() { @@ -2530,9 +2539,16 @@ func (t *App) createTable(creator CreateTableOptions) { } focusTable, ok := t.app.GetFocus().(*tview.Table) - if !ok || focusTable.GetTitle() != v.GetTitle() || tablesDiffer(focusTable, v) { - update() + isSameTable := ok && focusTable.GetTitle() == v.GetTitle() + if isSameTable && !tablesDiffer(focusTable, v) { + return + } + if isSameTable && !isSelectable { + // No selection to restore from t.position: carry the scroll offset over + // so a refresh does not send the reader back to the first row. + v.SetOffset(focusTable.GetOffset()) } + update() } func (t *App) selectedString() string { From bdfc710e9ddd3a548928adf142ba0e9bb9285c5f Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 08:53:56 +0200 Subject: [PATCH 39/54] Organize the tui views in a registry, with a navigation stack of frames Adding a view meant editing six places: the viewId const block, its String(), viewPrimitive(), the two switches of navFromTo() (teardown and setup) and the refresh switch of do(). They had already drifted apart: String() had no viewEvents case, so the events view had no head bar title, and viewPrimitive() had no case for any of the createTable based views, returning the object table for all of them. It was also dead code. Declare instead one viewDefs entry per view, holding its title and its enter, refresh and leave hooks. viewId.String(), the nav dispatch and the data update dispatch all read that single table, and a view owning a resource can no longer forget to release it in a switch case nobody remembers to extend. Four places built the flex layout by hand: navFromTo(), createTable(), listContexts() and the instance view postamble, and they disagreed. createTable() dropped the errors bar on every refresh, and so did the instance view. Funnel them all through mount(), which lays out the head bar, the optional banners, the view body and the errors bar, and remembers the layout so the help popup can restore it. The instance view restore was broken: it put the summary banner back in place of the resources table. This also removes the flex index guessing (GetItem(1) is the body, GetItem(2) is the instance table) from the popups and from the command line handler. The navigation stack now holds frames rather than bare view ids. A frame carries the cursor position and the element the view drilled down into, so: - coming back to a view lands where the user left it, instead of on the first cell: t.position was a single App field, shared by three views and reset on every navigation. - the pool drill down no longer needs previousSelectedElement: pushing a frame saves the parent element, popping restores it. - navigating to the frame already on top is a no-op, so hitting the log key twice no longer stacks two log frames, each needing its own ESC, while entering a pool from the pool list still pushes a frame. The stack is never empty: its root frame is the object view, or the context view for a user only granted the relay view, which is what backToContext was emulating. listContexts() becomes the context view enter hook, so t.focus() no longer lies while the context list is displayed. Also fixes, found on the way: - the instance view set two selection changed funcs on its table, and tview only keeps the last one: t.viewRID was never set, so the 't' and 'T' container terminal shortcuts never triggered. - entering a view left the head bar naming the previous one until the next cluster data change. - printf() added a second errors bar for 5s, then removed both. --- core/tui/instance.go | 30 +--- core/tui/keys.go | 4 +- core/tui/main.go | 367 ++++++----------------------------------- core/tui/network.go | 4 +- core/tui/objects.go | 5 +- core/tui/pool.go | 21 +-- core/tui/smoke_test.go | 149 +++++++++++++++++ core/tui/view.go | 366 ++++++++++++++++++++++++++++++++++++++++ core/tui/view_test.go | 224 +++++++++++++++++++++++++ 9 files changed, 820 insertions(+), 350 deletions(-) create mode 100644 core/tui/smoke_test.go create mode 100644 core/tui/view.go create mode 100644 core/tui/view_test.go diff --git a/core/tui/instance.go b/core/tui/instance.go index 8a57c2a97..7a06b3ee8 100644 --- a/core/tui/instance.go +++ b/core/tui/instance.go @@ -171,17 +171,6 @@ func (t *App) updateInstanceView() { table.SetEvaluateAllRows(true) table.SetSelectable(true, true) - table.SetSelectionChangedFunc(func(row, col int) { - t.viewRID = "" - if row == 0 { - return - } - if col == 0 { - t.viewRID = table.GetCell(row, col).Text - } - - }) - selectedFunc := func(row, col int) { cell := table.GetCell(row, col) rid := table.GetCell(row, 0).Text @@ -201,8 +190,13 @@ func (t *App) updateInstanceView() { table.SetSelectedFunc(selectedFunc) - table.SetSelectionChangedFunc(func(row, column int) { - t.position = Position{row: row, col: column} + // a single selection changed func: tview only keeps the last one set. + table.SetSelectionChangedFunc(func(row, col int) { + t.frame().position = Position{row: row, col: col} + t.viewRID = "" + if row > 0 && col == 0 { + t.viewRID = table.GetCell(row, col).Text + } }) setSelection := func(table *tview.Table) { @@ -263,14 +257,8 @@ func (t *App) updateInstanceView() { postamble := func() { t.cleanCommand() - - t.flex.Clear() - t.flex.AddItem(t.head, 1, 0, false) - t.flex.AddItem(table1, i+2, 0, false) - t.flex.AddItem(table, 0, 1, true) - t.app.SetFocus(table) - - table.Select(t.position.row, t.position.col) + t.mount(table, mountBanner{primitive: table1, height: i + 2}) + table.Select(t.frame().position.row, t.frame().position.col) } instanceState, ok := digest.Instances.ByNode()[t.viewNode] diff --git a/core/tui/keys.go b/core/tui/keys.go index c864bb15d..b0b4b604e 100644 --- a/core/tui/keys.go +++ b/core/tui/keys.go @@ -108,7 +108,9 @@ func (t *App) updateKeyTextView() { return } - t.initTextView() + if t.textView == nil { + return + } text := string(resp.Body) title := fmt.Sprintf("%s key %s", t.viewPath, t.viewKey) t.textView.SetTitle(title) diff --git a/core/tui/main.go b/core/tui/main.go index bdedd33ef..66295c13a 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -34,9 +34,6 @@ import ( ) type ( - viewId int - viewStack []viewId - Position struct { row int col int @@ -104,9 +101,12 @@ type ( lastDraw time.Time - selectedElement string - previousSelectedElement string - position Position + // selectedElement is the element the focused view drilled down + // into. Saved in and restored from the navigation stack frames. + selectedElement string + + // mounted is the layout of the focused view. + mounted mountSpec events chan event.Event stopEvents bool @@ -115,7 +115,6 @@ type ( isInEventView atomic.Bool isOnConfirmation bool - backToContext bool viewPath naming.Path viewNode string @@ -153,24 +152,6 @@ type ( } ) -const ( - viewObject viewId = iota - viewContext - viewConfig - viewKey - viewKeys - viewInstance - viewLog - viewPool - viewPoolVolume - viewNetwork - viewNetworkIpList - viewEvents - viewHbStatus - viewRelay - viewLast // marker, not a real view -) - var ( colorNone = tcell.ColorNone colorSelected = tcell.ColorDarkSlateGray @@ -180,9 +161,6 @@ var ( colorHead3 = tcell.NewHexColor(0x23415A) colorHighlight = tcell.ColorWhite - forceUpdate = true - updateIfChange = false - dataLostMessage = "I understand data will be lost." confLostMessage = "I understand the configuration will be lost." serviceInterruptionMessage = "I understand the selected services may be temporarily interrupted during failover, or durably interrupted if no failover is configured." @@ -203,26 +181,7 @@ func Run(options *Options) error { return app.Run() } -func (t viewStack) String() string { - l := []string{ - viewObject.String(), - } - for _, v := range t { - l = append(l, v.String()) - } - return strings.Join(l, " > ") -} - func (t *App) updateHead() { - type titler interface{ GetTitle() string } - if t.flex.GetItemCount() < 2 { - return - } - primitive := t.flex.GetItem(1) - box, ok := primitive.(titler) - if !ok { - return - } conn := func() string { endpoint := "" if t.client != nil { @@ -237,71 +196,19 @@ func (t *App) updateHead() { return fmt.Sprintf("%s@%s", t.user, endpoint) } } - title := box.GetTitle() + var title string + if box, ok := t.body().(interface{ GetTitle() string }); ok { + title = box.GetTitle() + } t.head.SetCell(0, 0, tview.NewTableCell(conn()).SetBackgroundColor(colorHead3)) t.head.SetCell(0, 1, tview.NewTableCell(" "+t.Frame.Current.Cluster.Config.Name).SetBackgroundColor(colorHead)) t.head.SetCell(0, 2, tview.NewTableCell(" "+title).SetBackgroundColor(colorHead2).SetExpansion(1)) } -func (t viewId) String() string { - switch t { - case viewObject: - return "objects" - case viewContext: - return "context" - case viewConfig: - return "configuration" - case viewKey: - return "key" - case viewKeys: - return "keys" - case viewInstance: - return "instance" - case viewLog: - return "log" - case viewPool: - return "pool" - case viewPoolVolume: - return "pool volume" - case viewNetwork: - return "network" - case viewNetworkIpList: - return "network ip list" - case viewHbStatus: - return "heartbeat status" - case viewRelay: - return "relay" - default: - return "" - } -} - -func (t *App) push(v viewId) { - t.stack = append(t.stack, v) -} - -func (t *App) pop() viewId { - n := len(t.stack) - if n == 0 { - return viewObject - } - v := t.stack[n-1] - t.stack = t.stack[:n-1] - return v -} - -func (t *App) focus() viewId { - n := len(t.stack) - if n == 0 { - return viewObject - } - return t.stack[n-1] -} - func NewApp(options *Options) *App { return &App{ - stack: make([]viewId, 0), + stack: viewStack{{id: viewObject}}, firstInstanceCol: 5, headerRightCol: 3, maxRetries: 600, @@ -321,9 +228,9 @@ func NewApp(options *Options) *App { } } +// resetSelected drops the selection of the focused view and returns the number +// of entries it dropped. func (t *App) resetSelected() int { - t.selectedElement = t.previousSelectedElement - t.previousSelectedElement = "" switch t.focus() { case viewInstance: n := len(t.selectedRIDs) @@ -351,17 +258,6 @@ func (t *App) initErrsTextView() { t.errs.SetBorder(false) } -func (t *App) viewPrimitive(v viewId) tview.Primitive { - switch v { - case viewConfig, viewInstance, viewKey, viewLog, viewEvents: - return t.textView - case viewKeys: - return t.keys - default: - return t.objects - } -} - func (t *App) initApp() { t.initHeadTextView() t.initObjectsTable() @@ -369,10 +265,8 @@ func (t *App) initApp() { t.app = tview.NewApplication() t.flex = tview.NewFlex().SetDirection(tview.FlexRow) - t.flex.AddItem(t.head, 1, 0, false) - t.updateHead() - t.flex.AddItem(t.objects, 0, 1, true) t.app.SetRoot(t.flex, true) + t.mount(t.objects) t.app.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { if t.command != nil { @@ -383,7 +277,7 @@ func (t *App) initApp() { } switch event.Key() { case tcell.KeyESC: - if n := t.resetSelected(); n > 0 || (t.Frame.Selector == "*/svc/*" && len(t.stack) == 0) { + if n := t.resetSelected(); n > 0 || (t.atRoot() && t.Frame.Selector == t.defaultSelector()) { return event } t.back() @@ -441,7 +335,9 @@ func (t *App) init() error { return nil } -func (t *App) listContexts() { +// updateContextList is the viewContext enter hook: it builds the context table +// and mounts it. +func (t *App) updateContextList() { cfg, err := clientcontext.Load() if err != nil { t.errorf("%s", err) @@ -502,37 +398,29 @@ func (t *App) listContexts() { t.errorf("%s", err) } else if resp, err := cli.GetAuthWhoAmIWithResponse(context.Background()); err != nil { t.errorf("%s", err) - t.listContexts() + t.navRoot(viewContext) } else if resp.StatusCode() == http.StatusOK { t.client = cli if streamClient, err := client.New(client.WithTimeout(0)); err != nil { t.errorf("new stream client: %s", err) - t.listContexts() + t.navRoot(viewContext) } else { t.streamClient = streamClient t.user = resp.JSON200.Name t.reconnect() - t.flex.Clear() - t.flex.AddItem(t.head, 1, 0, false) - t.flex.AddItem(t.objects, 0, 1, true) - t.app.SetFocus(t.objects) - t.updateHead() if resp.JSON200.RawGrant == "heartbeat" { + // this user is only granted the relay view: keep the + // context view as the stack root, so ESC comes back here. + t.resetStack(viewContext) t.nav(viewRelay) - t.backToContext = true - } else if t.backToContext { - t.backToContext = false - t.pop() + } else { + t.navRoot(viewObject) } } } }) - t.flex.Clear() - t.flex.AddItem(t.head, 1, 0, false) - t.flex.AddItem(v, 0, 1, true) - t.app.SetFocus(v) - t.updateHead() + t.mount(v) } func (t *App) Run() error { @@ -549,23 +437,26 @@ func (t *App) initContext() { t.errorf("%s", err) } else if resp, err := cli.GetAuthWhoAmIWithResponse(context.Background()); err != nil { t.errorf("%s", err) - t.listContexts() + t.navRoot(viewContext) } else if resp.StatusCode() == http.StatusOK { t.client = cli if streamClient, err := client.New(client.WithTimeout(0)); err != nil { t.errorf("new stream client: %s", err) - t.listContexts() + t.navRoot(viewContext) } else { t.streamClient = streamClient t.user = resp.JSON200.Name t.reconnect() if resp.JSON200.RawGrant == "heartbeat" { + // this user is only granted the relay view: root the + // navigation stack on the context view so ESC offers to + // connect elsewhere. + t.resetStack(viewContext) t.nav(viewRelay) - t.backToContext = true } } } else { - t.listContexts() + t.navRoot(viewContext) } } @@ -650,31 +541,7 @@ func (t *App) do(evReader event.ReadCloser) error { t.eventCount++ t.app.QueueUpdateDraw(func() { // TODO: detect if t.updateInstanceView and t.updateConfigView need to be called (config mtime change, ...) - t.updateHead() - switch t.focus() { - case viewInstance: - t.updateInstanceView() - case viewConfig: - t.updateConfigView() - case viewKeys: - t.updateKeysView() - case viewPool: - t.updatePoolList(updateIfChange) - case viewPoolVolume: - t.updatePoolVolume(t.selectedElement) - case viewNetwork: - t.updateNetworkList() - case viewNetworkIpList: - t.updateNetworkIpList(t.selectedElement) - case viewEvents: - t.updateEventsView() - case viewHbStatus: - t.updateHbStatus() - case viewRelay: - t.updateRelayStatus() - default: - t.updateObjects() - } + t.refreshView() }) } }(data.DeepCopy()) @@ -863,11 +730,13 @@ func (t *App) isNodeSelected(node string) bool { } func (t *App) cleanCommand() { + // the command input took the errors bar place: give it back t.flex.RemoveItem(t.command) + t.flex.AddItem(t.errs, 1, 0, false) t.command = nil t.focused = false if !t.isOnConfirmation { - t.app.SetFocus(t.flex.GetItem(1)) + t.app.SetFocus(t.body()) } } @@ -1105,7 +974,7 @@ func (t *App) onRuneColumn(event *tcell.EventKey) { row, col := t.objects.GetSelection() switch { case t.focus() == viewInstance && row > 1: - if table, ok := t.flex.GetItem(2).(*tview.Table); ok { + if table, ok := t.body().(*tview.Table); ok { row, col := table.GetSelection() rid := table.GetCell(row, col).Text selection := make(map[[3]string]any) @@ -1176,7 +1045,7 @@ func (t *App) confirmAction(action func(), messages ...string) { if oldFocus != nil && t.focus() != viewObject { t.app.SetFocus(oldFocus) } else { - t.app.SetFocus(t.flex.GetItem(1)) + t.app.SetFocus(t.body()) } t.isOnConfirmation = false t.focused = false @@ -1928,7 +1797,7 @@ func (t *App) onRuneH(event *tcell.EventKey) { return } - savedItem := t.flex.GetItem(1) + savedMount := t.mounted savedFocus := t.app.GetFocus() v := tview.NewTextView(). @@ -1941,9 +1810,7 @@ func (t *App) onRuneH(event *tcell.EventKey) { switch event.Key() { case tcell.KeyESC: t.help = nil - t.flex.RemoveItem(v) - t.flex.AddItem(t.head, 1, 0, false) - t.flex.AddItem(savedItem, 0, 1, true) + t.remount(savedMount) t.app.SetFocus(savedFocus) } return event @@ -1975,12 +1842,6 @@ func (t *App) updateLogTextView() { }) t.textView.Clear() - lines := 50 - follow := true - - // Create the output writer for the TUI text view - outputWriter := tview.ANSIWriter(t.textView) - var nodes []string if t.viewNode != "" { // instance or node logs @@ -2000,6 +1861,12 @@ func (t *App) updateLogTextView() { } } + lines := 50 + follow := true + + // Create the output writer for the TUI text view + outputWriter := tview.ANSIWriter(t.textView) + // Create streams for all nodes streams := make([]logreader.NodeStream, 0, len(nodes)) for i, node := range nodes { @@ -2152,8 +2019,6 @@ func (t *App) onRuneSlash(event *tcell.EventKey) { } func (t *App) onRuneC(event *tcell.EventKey) { - t.initTextView() - t.updateConfigView() t.nav(viewConfig) } @@ -2338,126 +2203,15 @@ func (t *App) errorf(format string, args ...any) { } func (t *App) printf(color tcell.Color, format string, args ...any) { - t.flex.AddItem(t.errs, 1, 0, false) t.errs.Clear() t.errs.SetBackgroundColor(color) fmt.Fprintf(t.errs, format, args...) time.AfterFunc(5*time.Second, func() { - t.flex.RemoveItem(t.errs) + t.errs.Clear() + t.errs.SetBackgroundColor(colorNone) }) } -func (t *App) nav(to viewId) { - from := t.focus() - if t.backToContext && to == viewRelay { - t.navFromTo(from, to) - return - } - t.push(to) - if to == from { - return - } - t.navFromTo(from, to) -} - -func (t *App) back() { - if t.backToContext { - if t.focus() == viewContext { - return - } - t.listContexts() - return - } - if t.resetSelected() == 0 && len(t.stack) == 0 && !t.focused { - filter := "*/svc/*" - if t.options != nil && t.options.Selector != "" { - filter = t.options.Selector - } - t.setFilter(filter) - return - } - from := t.pop() - to := t.focus() - t.navFromTo(from, to) -} - -func (t *App) navFromTo(from, to viewId) { - t.flex.Clear() - t.flex.AddItem(t.head, 1, 0, false) - t.lastUpdatedAt = time.Time{} - t.position = Position{row: 0, col: 0} - switch from { - case viewObject: - case viewLog: - t.textView.SetChangedFunc(nil) - t.textView = nil - t.logCloser.CloseAll() - case viewConfig, viewInstance, viewKey: - t.textView = nil - case viewKeys: - t.keys = nil - case viewEvents: - t.textView = nil - if t.eventsCancel != nil { - t.eventsCancel() - } - t.isInEventView.Store(false) - } - switch to { - case viewContext: - t.listContexts() - case viewLog: - t.initTextView() - t.flex.AddItem(t.textView, 0, 1, true) - t.app.SetFocus(t.textView) - t.updateLogTextView() - case viewConfig: - t.initTextView() - t.flex.AddItem(t.textView, 0, 1, true) - t.app.SetFocus(t.textView) - case viewKey: - t.initTextView() - t.flex.AddItem(t.textView, 0, 1, true) - t.app.SetFocus(t.textView) - t.updateKeyTextView() - case viewInstance: - t.initTextView() - t.flex.AddItem(t.textView, 0, 1, true) - t.app.SetFocus(t.textView) - t.updateInstanceView() - case viewKeys: - t.initKeysTable() - t.flex.AddItem(t.keys, 0, 1, true) - t.app.SetFocus(t.keys) - t.updateKeysView() - case viewObject: - t.flex.AddItem(t.objects, 0, 1, true) - t.app.SetFocus(t.objects) - t.updateObjects() - case viewNetwork: - t.updateNetworkList() - case viewPool: - t.updatePoolList(forceUpdate) - case viewNetworkIpList: - t.updateNetworkIpList(t.selectedElement) - case viewPoolVolume: - t.updatePoolVolume(t.selectedElement) - case viewEvents: - t.isInEventView.Store(true) - t.initTextView() - t.initEventsView() - t.flex.AddItem(t.textView, 0, 1, true) - t.app.SetFocus(t.textView) - t.updateEventsView() - case viewHbStatus: - t.updateHbStatus() - case viewRelay: - t.updateRelayStatus() - } - t.updateHead() - t.flex.AddItem(t.errs, 1, 0, false) -} - func (t *App) createTable(creator CreateTableOptions) { if t.focused { return @@ -2474,11 +2228,7 @@ func (t *App) createTable(creator CreateTableOptions) { v.SetTitle(creator.title) update := func() { - t.flex.Clear() - t.flex.AddItem(t.head, 1, 0, false) - t.flex.AddItem(v, 0, 1, true) - t.app.SetFocus(v) - t.updateHead() + t.mount(v) } for i, title := range creator.titles { @@ -2496,7 +2246,7 @@ func (t *App) createTable(creator CreateTableOptions) { }) v.SetSelectionChangedFunc(func(row, column int) { - t.position = Position{row: row, col: column} + t.frame().position = Position{row: row, col: column} }) for i, elements := range creator.elementsList { @@ -2508,7 +2258,7 @@ func (t *App) createTable(creator CreateTableOptions) { } if isSelectable { - v.Select(t.position.row, t.position.col) + v.Select(t.frame().position.row, t.frame().position.col) } tablesDiffer := func(a, b *tview.Table) bool { @@ -2544,7 +2294,7 @@ func (t *App) createTable(creator CreateTableOptions) { return } if isSameTable && !isSelectable { - // No selection to restore from t.position: carry the scroll offset over + // No selection to restore from the frame cursor: carry the scroll offset over // so a refresh does not send the reader back to the first row. v.SetOffset(focusTable.GetOffset()) } @@ -2567,17 +2317,6 @@ func (t *App) selectedString() string { } } -func (t *App) initTextView() { - if t.textView != nil { - return - } - v := tview.NewTextView() - v.SetScrollable(true) - v.SetBorder(false) - t.textView = v - return -} - func (t *App) reconnect() { select { case t.restartC <- nil: diff --git a/core/tui/network.go b/core/tui/network.go index ad271a60e..4f82fa623 100644 --- a/core/tui/network.go +++ b/core/tui/network.go @@ -92,7 +92,9 @@ func (t *App) updateNetworkList() { }) } -func (t *App) updateNetworkIpList(name string) { +// updateNetworkIps is the viewNetworkIpList enter and refresh hook. +func (t *App) updateNetworkIps() { + name := t.selectedElement title := fmt.Sprintf("Network %s IPs", name) titles := []string{"OBJECT", "NODE", "RID", "IP", "NET_NAME", "NET_TYPE"} var elementsList [][]string diff --git a/core/tui/objects.go b/core/tui/objects.go index d027b072b..fd7bca2bf 100644 --- a/core/tui/objects.go +++ b/core/tui/objects.go @@ -24,12 +24,11 @@ func (t *App) initObjectsTable() { row, col := table.GetSelection() switch { case !t.viewPath.IsZero() && t.viewNode != "" && !(t.viewPath.Kind == naming.KindCfg || t.viewPath.Kind == naming.KindSec): - t.initTextView() t.nav(viewInstance) case t.viewPath.Kind == naming.KindCfg || t.viewPath.Kind == naming.KindSec: t.nav(viewKeys) case row == 0 && col == 1: - t.listContexts() + t.nav(viewContext) case row == 1 && col == 1: t.nav(viewEvents) case (row >= hbIndexRow && row <= hbIndexRow+2) && (col >= t.headerRightCol && col <= t.firstInstanceCol+len(t.Current.Cluster.Config.Nodes)-1): @@ -107,7 +106,7 @@ func (t *App) initObjectsTable() { if col >= t.firstInstanceCol { t.viewNode = t.objects.GetCell(0, col).Text } - t.position = Position{row: row, col: col} + t.frame().position = Position{row: row, col: col} handleCursorPosition(row, col) }) table.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { diff --git a/core/tui/pool.go b/core/tui/pool.go index 021d313b2..e6669d3b8 100644 --- a/core/tui/pool.go +++ b/core/tui/pool.go @@ -75,8 +75,13 @@ func (t *App) skipIfPoolNotUpdated() bool { return true } -func (t *App) updatePoolList(forceUpdate bool) { - if !forceUpdate && t.skipIfPoolNotUpdated() { +// updatePools is the viewPool enter and refresh hook. It lists all the pools, +// or, when the view drilled down into one, the per node status of that pool. +func (t *App) updatePools() { + if t.skipIfPoolNotUpdated() && !t.lastUpdatedAt.IsZero() { + // No pool data change to paint. A still zero lastUpdatedAt means the + // view was just entered and no node has published pool data yet: + // paint the empty table, so the previous view is not left on screen. return } title := "pools" @@ -144,15 +149,9 @@ func (t *App) updatePoolList(forceUpdate bool) { if row == 0 { break } - poolName := v.GetCell(row, 0).Text - if col == 0 && t.selectedElement != "" { - t.previousSelectedElement = t.selectedElement - } - t.selectedElement = poolName + t.selectedElement = v.GetCell(row, 0).Text if col == 0 { t.nav(viewPool) - t.position = Position{row: 0, col: 0} - t.updatePoolList(forceUpdate) } else if col == 4 { t.nav(viewPoolVolume) } @@ -162,7 +161,9 @@ func (t *App) updatePoolList(forceUpdate bool) { }) } -func (t *App) updatePoolVolume(name string) { +// updatePoolVolumes is the viewPoolVolume enter and refresh hook. +func (t *App) updatePoolVolumes() { + name := t.selectedElement title := fmt.Sprintf("%s volumes", name) titles := []string{"POOL", "PATH", "SIZE", "CHILDREN", "IS_ORPHAN"} var elementsList [][]string diff --git a/core/tui/smoke_test.go b/core/tui/smoke_test.go new file mode 100644 index 000000000..1ca149996 --- /dev/null +++ b/core/tui/smoke_test.go @@ -0,0 +1,149 @@ +package tui + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" + + "github.com/opensvc/om3/v3/core/client" +) + +// TestSmokeLiveDaemon drives the real application against the daemon of the +// node it runs on: it enters every view, pages through it and comes back to +// the object view. Skipped when no daemon answers. +func TestSmokeLiveDaemon(t *testing.T) { + cli, err := client.New() + if err != nil { + t.Skipf("no client: %s", err) + } + if resp, err := cli.GetAuthWhoAmIWithResponse(context.Background()); err != nil { + t.Skipf("no daemon: %s", err) + } else if resp.StatusCode() != http.StatusOK { + t.Skipf("no daemon: %s", resp.Status()) + } + + screen := tcell.NewSimulationScreen("UTF-8") + if err := screen.Init(); err != nil { + t.Fatal(err) + } + screen.SetSize(160, 12) // short on purpose: more hb lines than term lines + + a := NewApp(nil) + if err := a.init(); err != nil { + t.Fatal(err) + } + a.app.SetScreen(screen) + go a.runEventReader() + a.initContext() + + done := make(chan struct{}) + go func() { defer close(done); _ = a.app.Run() }() + defer func() { + a.stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("the application did not stop") + } + }() + + readScreen := func() string { + var b strings.Builder + w, h := screen.Size() + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + r, _, _, _ := screen.GetContent(x, y) + b.WriteRune(r) + } + b.WriteString("\n") + } + return b.String() + } + + // let the first cluster data land + time.Sleep(2 * time.Second) + t.Logf("nodes=%d objects=%d", len(a.Current.Cluster.Config.Nodes), len(a.Current.Cluster.Object)) + + var dump func() string + sync := func(f func()) { + ch := make(chan struct{}) + a.app.QueueUpdateDraw(func() { f(); close(ch) }) + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("the event loop is wedged") + } + } + // read the screen from the application loop, so the dump never catches a + // repaint halfway through + dump = func() string { + var s string + sync(func() { s = readScreen() }) + return s + } + key := func(k tcell.Key) { + screen.InjectKey(k, 0, tcell.ModNone) + sync(func() {}) + } + + for _, v := range []viewId{viewHbStatus, viewPool, viewNetwork, viewRelay, viewEvents, viewConfig, viewLog} { + sync(func() { a.nav(v) }) + time.Sleep(300 * time.Millisecond) + if got := a.focus(); got != v { + t.Fatalf("nav to %s: focused on %s", v, got) + } + for i := 0; i < 3; i++ { + key(tcell.KeyPgDn) + } + key(tcell.KeyPgUp) + key(tcell.KeyDown) + if n := a.flex.GetItemCount(); n < 3 { + t.Errorf("%s: only %d flex items", v, n) + } + head := strings.TrimSpace(strings.SplitN(dump(), "\n", 2)[0]) + t.Logf("%-18s head=%q stack=%s", v, head, a.stack) + if head == "" { + t.Errorf("%s: the head bar is empty", v) + } + sync(func() { a.back() }) + time.Sleep(200 * time.Millisecond) + if !a.atRoot() || a.focus() != viewObject { + t.Fatalf("back from %s: stack is %s", v, a.stack) + } + } + // drill down into the first pool, and back out + sync(func() { a.nav(viewPool) }) + time.Sleep(300 * time.Millisecond) + table, ok := a.body().(*tview.Table) + if !ok { + t.Fatalf("the pool view body is a %T", a.body()) + } + if table.GetRowCount() < 2 { + t.Skip("no pool to drill down into") + } + sync(func() { table.Select(1, 0) }) + poolName := table.GetCell(1, 0).Text + key(tcell.KeyEnter) + time.Sleep(300 * time.Millisecond) + if a.focus() != viewPool || len(a.stack) != 3 || a.selectedElement != poolName { + t.Fatalf("drilling into pool %q: stack=%s element=%q", poolName, a.stack, a.selectedElement) + } + t.Logf("drilled into pool %q, head=%q", poolName, strings.TrimSpace(strings.SplitN(dump(), "\n", 2)[0])) + + sync(func() { a.back() }) + time.Sleep(300 * time.Millisecond) + if a.focus() != viewPool || len(a.stack) != 2 || a.selectedElement != "" { + t.Fatalf("back to the pool list: stack=%s element=%q", a.stack, a.selectedElement) + } + sync(func() { a.back() }) + if !a.atRoot() { + t.Fatalf("back to the object view: stack=%s", a.stack) + } + + t.Logf("final screen:\n%s", dump()) +} diff --git a/core/tui/view.go b/core/tui/view.go new file mode 100644 index 000000000..f375260c7 --- /dev/null +++ b/core/tui/view.go @@ -0,0 +1,366 @@ +package tui + +import ( + "strings" + "time" + + "github.com/rivo/tview" +) + +type ( + // viewId identifies a view. Every view must have an entry in viewDefs. + viewId int + + // viewDef describes the lifecycle of a view. It is the single place where + // a view declares its name, how it puts itself on screen, how it keeps + // itself up to date and what it has to release when the user leaves it. + viewDef struct { + // title names the view in the head bar and in the navigation stack. + title string + + // enter builds the view primitive, mounts it and populates it. It is + // called when the view gets the focus, either by navigating forward + // with nav() or backward with back(). + enter func(*App) + + // refresh repopulates the view on a cluster data update. Left nil by + // the views that don't depend on the cluster data, or that stream + // their own content. + refresh func(*App) + + // leave releases what enter and refresh acquired: readers, contexts + // and cached primitives. Left nil by the views owning no resource. + leave func(*App) + } + + // frame is a navigation stack entry: a view plus the state the user + // expects to find again when navigating back to it. + frame struct { + id viewId + + // position is the cursor of the view table. + position Position + + // selectedElement is the element the view drilled down into, ie the + // pool name of a pool volumes view. Inherited from the parent frame + // on push, restored on pop. + selectedElement string + } + + viewStack []frame + + // mountBanner is a fixed height primitive displayed between the head bar + // and the view body. + mountBanner struct { + primitive tview.Primitive + height int + } + + // mountSpec is the layout of a mounted view, remembered so the layout can + // be restored after a full screen popup like the help. + mountSpec struct { + body tview.Primitive + banners []mountBanner + } +) + +const ( + viewObject viewId = iota + viewContext + viewConfig + viewKey + viewKeys + viewInstance + viewLog + viewPool + viewPoolVolume + viewNetwork + viewNetworkIpList + viewEvents + viewHbStatus + viewRelay +) + +// viewDefs is the view registry. Adding a view is adding an entry here. +// +// The views whose enter and refresh go through createTable() mount themselves: +// createTable() calls mount() when it has to replace the displayed table. +var viewDefs map[viewId]viewDef + +func init() { + viewDefs = map[viewId]viewDef{ + viewObject: { + title: "objects", + enter: func(t *App) { + t.mount(t.objects) + t.updateObjects() + }, + refresh: (*App).updateObjects, + }, + viewContext: { + title: "context", + enter: (*App).updateContextList, + }, + viewConfig: { + title: "configuration", + enter: func(t *App) { + t.mountTextView() + t.updateConfigView() + }, + refresh: (*App).updateConfigView, + leave: (*App).releaseTextView, + }, + viewKey: { + title: "key", + enter: func(t *App) { + t.mountTextView() + t.updateKeyTextView() + }, + leave: (*App).releaseTextView, + }, + viewKeys: { + title: "keys", + enter: func(t *App) { + t.initKeysTable() + t.mount(t.keys) + t.updateKeysView() + }, + refresh: (*App).updateKeysView, + leave: func(t *App) { t.keys = nil }, + }, + viewInstance: { + title: "instance", + enter: func(t *App) { + // updateInstanceView() mounts its own primitives, but it bails + // out when the instance data is not there yet: mount a body so + // the screen is never left without one. + t.mountTextView() + t.updateInstanceView() + }, + refresh: (*App).updateInstanceView, + leave: (*App).releaseTextView, + }, + viewLog: { + title: "log", + enter: func(t *App) { + t.mountTextView() + // updateLogTextView() opens the log readers and lets them stream + // into the text view: it must not be called on data updates. + t.updateLogTextView() + }, + leave: func(t *App) { + if t.textView != nil { + t.textView.SetChangedFunc(nil) + } + t.releaseTextView() + t.logCloser.CloseAll() + }, + }, + viewEvents: { + title: "events", + enter: func(t *App) { + t.isInEventView.Store(true) + t.mountTextView() + t.initEventsView() + t.updateEventsView() + }, + refresh: (*App).updateEventsView, + leave: func(t *App) { + t.isInEventView.Store(false) + if t.eventsCancel != nil { + t.eventsCancel() + } + t.releaseTextView() + }, + }, + viewPool: { + title: "pool", + enter: (*App).updatePools, + refresh: (*App).updatePools, + }, + viewPoolVolume: { + title: "pool volume", + enter: (*App).updatePoolVolumes, + refresh: (*App).updatePoolVolumes, + }, + viewNetwork: { + title: "network", + enter: (*App).updateNetworkList, + refresh: (*App).updateNetworkList, + }, + viewNetworkIpList: { + title: "network ip list", + enter: (*App).updateNetworkIps, + refresh: (*App).updateNetworkIps, + }, + viewHbStatus: { + title: "heartbeat status", + enter: (*App).updateHbStatus, + refresh: (*App).updateHbStatus, + }, + viewRelay: { + title: "relay", + enter: (*App).updateRelayStatus, + refresh: (*App).updateRelayStatus, + }, + } +} + +func (t viewId) String() string { + return viewDefs[t].title +} + +func (t viewStack) String() string { + l := make([]string, len(t)) + for i, f := range t { + l[i] = f.id.String() + } + return strings.Join(l, " > ") +} + +// +// Navigation stack +// + +// frame returns the focused stack frame. The stack is never empty: its first +// frame is the root view, the one ESC can not pop. +func (t *App) frame() *frame { + return &t.stack[len(t.stack)-1] +} + +// focus returns the id of the focused view. +func (t *App) focus() viewId { + return t.frame().id +} + +// atRoot returns true when the focused view is the one ESC can not pop. +func (t *App) atRoot() bool { + return len(t.stack) == 1 +} + +// resetStack makes v the only frame of the navigation stack, without entering +// it. +func (t *App) resetStack(v viewId) { + t.stack = viewStack{{id: v}} +} + +// nav pushes a new frame on the navigation stack and enters it. +// +// Navigating to the frame already on top is a no-op: a view is identified by +// its id and by the element it drilled down into, so that entering a pool from +// the pool list does push a frame, while hitting the log key twice does not. +func (t *App) nav(to viewId) { + if f := t.frame(); f.id == to && f.selectedElement == t.selectedElement { + return + } + t.leaveView(t.focus()) + t.stack = append(t.stack, frame{id: to, selectedElement: t.selectedElement}) + t.enterView(to) +} + +// navRoot enters v and makes it the root of a new navigation stack. +func (t *App) navRoot(v viewId) { + t.leaveView(t.focus()) + t.resetStack(v) + t.enterView(v) +} + +// back pops the focused frame and re-enters the one below, restoring the +// element it had drilled down into. +// +// At the root of the stack there is nothing to pop: the object view then +// resets its selector to the one asked on the command line. +func (t *App) back() { + if t.atRoot() { + if t.focus() == viewObject { + t.setFilter(t.defaultSelector()) + } + return + } + t.leaveView(t.focus()) + t.stack = t.stack[:len(t.stack)-1] + t.selectedElement = t.frame().selectedElement + t.enterView(t.focus()) +} + +func (t *App) enterView(v viewId) { + t.lastUpdatedAt = time.Time{} + if enter := viewDefs[v].enter; enter != nil { + enter(t) + } + // enter() sets the view title after mounting: refresh the head bar so it + // names the view without waiting for the next cluster data update. + t.updateHead() +} + +func (t *App) leaveView(v viewId) { + if leave := viewDefs[v].leave; leave != nil { + leave(t) + } +} + +// refreshView repopulates the focused view. Called on every cluster data +// update. +func (t *App) refreshView() { + if t.help != nil { + // don't pull the help popup from under the reader + return + } + if refresh := viewDefs[t.focus()].refresh; refresh != nil { + refresh(t) + } + t.updateHead() +} + +// defaultSelector returns the object selector the object view falls back to. +func (t *App) defaultSelector() string { + if t.options != nil && t.options.Selector != "" { + return t.options.Selector + } + return "*/svc/*" +} + +// +// Layout +// + +// mount lays the application out: the head bar on top, the optional fixed +// height banners, the view body taking the remaining height and the focus, +// and the errors bar at the bottom. +func (t *App) mount(body tview.Primitive, banners ...mountBanner) { + t.remount(mountSpec{body: body, banners: banners}) +} + +func (t *App) remount(spec mountSpec) { + t.mounted = spec + t.flex.Clear() + t.flex.AddItem(t.head, 1, 0, false) + for _, banner := range spec.banners { + t.flex.AddItem(banner.primitive, banner.height, 0, false) + } + t.flex.AddItem(spec.body, 0, 1, true) + t.flex.AddItem(t.errs, 1, 0, false) + t.app.SetFocus(spec.body) + t.updateHead() +} + +// body returns the mounted view body, the primitive to give the focus back to +// after a popup. +func (t *App) body() tview.Primitive { + return t.mounted.body +} + +// mountTextView mounts the shared text view, creating it if needed. +func (t *App) mountTextView() { + if t.textView == nil { + v := tview.NewTextView() + v.SetScrollable(true) + v.SetBorder(false) + t.textView = v + } + t.mount(t.textView) +} + +func (t *App) releaseTextView() { + t.textView = nil +} diff --git a/core/tui/view_test.go b/core/tui/view_test.go new file mode 100644 index 000000000..d4004c99e --- /dev/null +++ b/core/tui/view_test.go @@ -0,0 +1,224 @@ +package tui + +import ( + "testing" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +// newTestApp returns an App laid out on a simulation screen, without running +// the event loop. +func newTestApp(t *testing.T) *App { + t.Helper() + screen := tcell.NewSimulationScreen("UTF-8") + if err := screen.Init(); err != nil { + t.Fatal(err) + } + screen.SetSize(120, 24) + + a := NewApp(nil) + a.initHeadTextView() + a.initObjectsTable() + a.initErrsTextView() + a.app = tview.NewApplication().SetScreen(screen) + a.flex = tview.NewFlex().SetDirection(tview.FlexRow) + a.app.SetRoot(a.flex, true) + a.mount(a.objects) + return a +} + +// stubViews replaces the view registry with views recording their enter and +// leave calls, and restores the real registry when the test ends. +func stubViews(t *testing.T, log *[]string, ids ...viewId) { + t.Helper() + saved := viewDefs + t.Cleanup(func() { viewDefs = saved }) + + viewDefs = make(map[viewId]viewDef, len(ids)) + for _, id := range ids { + name := saved[id].title + viewDefs[id] = viewDef{ + title: name, + enter: func(*App) { *log = append(*log, "enter "+name) }, + leave: func(*App) { *log = append(*log, "leave "+name) }, + } + } +} + +// Every view id must be declared in the registry: viewId.String() and the +// enter, leave and refresh dispatches all go through it. +func TestViewDefsAreComplete(t *testing.T) { + for id := viewObject; id <= viewRelay; id++ { + def, ok := viewDefs[id] + if !ok { + t.Errorf("view id %d has no viewDefs entry", int(id)) + continue + } + if def.title == "" { + t.Errorf("view id %d has no title", int(id)) + } + if def.enter == nil { + t.Errorf("view %s has no enter hook", def.title) + } + } +} + +func TestNavStack(t *testing.T) { + var log []string + stubViews(t, &log, viewObject, viewPool, viewPoolVolume, viewLog) + a := newTestApp(t) + + if !a.atRoot() || a.focus() != viewObject { + t.Fatalf("a new app must be rooted on the object view, got %s", a.stack) + } + + a.nav(viewLog) + if a.focus() != viewLog || len(a.stack) != 2 { + t.Fatalf("nav to the log view: got %s", a.stack) + } + + // hitting the log key again must not stack a second log frame + a.nav(viewLog) + if len(a.stack) != 2 { + t.Fatalf("nav to the focused view must be a no-op, got %s", a.stack) + } + + a.back() + if a.focus() != viewObject || !a.atRoot() { + t.Fatalf("back to the object view: got %s", a.stack) + } + + if want := "leave objects enter log leave log enter objects"; join(log) != want { + t.Fatalf("hooks: got %q, want %q", join(log), want) + } +} + +// Drilling down keeps one frame per drilled element, and coming back restores +// the element of the frame below. +func TestNavStackDrillDown(t *testing.T) { + var log []string + stubViews(t, &log, viewObject, viewPool, viewPoolVolume) + a := newTestApp(t) + + a.nav(viewPool) // the pool list + a.selectedElement = "pool1" + a.nav(viewPool) // that pool, per node + if len(a.stack) != 3 { + t.Fatalf("drilling down into a pool must push a frame, got %s", a.stack) + } + + // re-entering the same pool must not stack a second frame + a.nav(viewPool) + if len(a.stack) != 3 { + t.Fatalf("re-entering the same pool must be a no-op, got %s", a.stack) + } + + a.nav(viewPoolVolume) + if a.focus() != viewPoolVolume || a.selectedElement != "pool1" { + t.Fatalf("the volume view must inherit the drilled pool, got %q", a.selectedElement) + } + + a.back() + if a.focus() != viewPool || a.selectedElement != "pool1" { + t.Fatalf("back to the pool detail: got %s %q", a.stack, a.selectedElement) + } + + a.back() + if a.focus() != viewPool || a.selectedElement != "" { + t.Fatalf("back to the pool list must clear the drilled pool, got %s %q", a.stack, a.selectedElement) + } + + a.back() + if !a.atRoot() || a.focus() != viewObject { + t.Fatalf("back to the object view: got %s", a.stack) + } +} + +// The cursor is remembered per frame, so coming back to a view lands where the +// user left it. +func TestNavStackKeepsCursorPerFrame(t *testing.T) { + var log []string + stubViews(t, &log, viewObject, viewInstance, viewLog) + a := newTestApp(t) + + a.nav(viewInstance) + a.frame().position = Position{row: 12, col: 3} + + a.nav(viewLog) + if got := a.frame().position; got != (Position{}) { + t.Fatalf("a new frame must start at the first cell, got %v", got) + } + + a.back() + if got := a.frame().position; got != (Position{row: 12, col: 3}) { + t.Fatalf("back must restore the frame cursor, got %v", got) + } +} + +// At the root of the stack there is nothing to pop. +func TestBackAtRoot(t *testing.T) { + var log []string + stubViews(t, &log, viewObject) + a := newTestApp(t) + + a.back() + if !a.atRoot() { + t.Fatalf("back at the root must not pop, got %s", a.stack) + } +} + +// navRoot drops the whole stack, leaving and entering as usual. +func TestNavRoot(t *testing.T) { + var log []string + stubViews(t, &log, viewObject, viewContext, viewRelay) + a := newTestApp(t) + + a.nav(viewRelay) + a.navRoot(viewContext) + if !a.atRoot() || a.focus() != viewContext { + t.Fatalf("navRoot must leave a single frame, got %s", a.stack) + } + if want := "leave objects enter relay leave relay enter context"; join(log) != want { + t.Fatalf("hooks: got %q, want %q", join(log), want) + } +} + +// mount lays the head bar, the banners, the body and the errors bar out, and +// remount restores the whole layout, banners included. +func TestMountLayout(t *testing.T) { + a := newTestApp(t) + + banner := tview.NewTable() + body := tview.NewTable() + a.mount(body, mountBanner{primitive: banner, height: 4}) + + if got, want := a.flex.GetItemCount(), 4; got != want { + t.Fatalf("mounted item count: got %d, want %d", got, want) + } + if a.flex.GetItem(0) != a.head || a.flex.GetItem(1) != banner || + a.flex.GetItem(2) != body || a.flex.GetItem(3) != a.errs { + t.Fatal("mounted items are not head, banner, body, errs") + } + if a.body() != body { + t.Fatal("body() must return the mounted body") + } + + saved := a.mounted + a.mount(tview.NewTable()) + a.remount(saved) + if a.flex.GetItemCount() != 4 || a.flex.GetItem(1) != banner || a.body() != body { + t.Fatal("remount must restore the banners and the body") + } +} + +func join(l []string) string { + s := "" + for i, e := range l { + if i > 0 { + s += " " + } + s += e + } + return s +} From 3cff5b0dc3e8eba10e28aa6dfedf983cc2adbf15 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 08:54:32 +0200 Subject: [PATCH 40/54] Fix the tui deadlock on entering the log view Entering the log view opens one log reader per node, a blocking daemon call, and it ran on the tview loop. That call is served on the same connection as the event stream, and the only reader of that stream is do(), which is itself blocked in QueueUpdateDraw waiting for the tview loop to run the update. The connection stops being drained, its flow control window fills, the log request never completes, and the whole application is wedged. Open the readers from a goroutine instead, and hand it the node list and the object path it needs, so it does not race with the view fields. A reader can now be opened after the user has already left the log view and CloseAll() has run. Have AtomicCloserSlice refuse a closer in that case, and let the caller close it, instead of leaking a followed log stream. The log view enter hook re-arms the slice. --- core/tui/closers.go | 22 ++++++++++++++++++++-- core/tui/main.go | 44 ++++++++++++++++++++++++++++---------------- core/tui/view.go | 1 + 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/core/tui/closers.go b/core/tui/closers.go index f2f3dfd26..9f023cb6e 100644 --- a/core/tui/closers.go +++ b/core/tui/closers.go @@ -8,13 +8,30 @@ import ( type AtomicCloserSlice struct { mu sync.RWMutex closers []io.Closer + + // closed is set by CloseAll and cleared by Reset. It makes Append reject + // the closers opened by a goroutine that lost the race with CloseAll. + closed bool } -// Append adds a closer to the slice thread-safely. -func (a *AtomicCloserSlice) Append(c io.Closer) { +// Append adds a closer to the slice thread-safely. It returns false, and does +// not take ownership of c, when CloseAll was called since the last Reset: the +// caller is expected to close c itself. +func (a *AtomicCloserSlice) Append(c io.Closer) bool { a.mu.Lock() defer a.mu.Unlock() + if a.closed { + return false + } a.closers = append(a.closers, c) + return true +} + +// Reset re-arms the slice: Append accepts closers again. +func (a *AtomicCloserSlice) Reset() { + a.mu.Lock() + defer a.mu.Unlock() + a.closed = false } // Get returns the slice thread-safely. @@ -38,6 +55,7 @@ func (a *AtomicCloserSlice) CloseAll() error { } } a.closers = nil // Clear the slice after closing. + a.closed = true if len(errs) > 0 { return errs[0] // Or use errors.Join(errs...) in Go 1.20+ } diff --git a/core/tui/main.go b/core/tui/main.go index 66295c13a..f35dcef72 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "net/http" "os" "os/exec" @@ -1861,20 +1862,26 @@ func (t *App) updateLogTextView() { } } + // Opening the log readers is a blocking daemon call, served on the + // connection the event stream is served on. That stream is only drained by + // the goroutine feeding QueueUpdateDraw, which waits for the tview loop: + // opening the readers here would deadlock the application. + go t.streamLogs(nodes, t.viewPath, tview.ANSIWriter(t.textView)) +} + +// streamLogs opens a log reader per node and streams them, merged, into w. It +// runs outside of the tview loop. +func (t *App) streamLogs(nodes []string, path naming.Path, w io.Writer) { lines := 50 follow := true - // Create the output writer for the TUI text view - outputWriter := tview.ANSIWriter(t.textView) - - // Create streams for all nodes streams := make([]logreader.NodeStream, 0, len(nodes)) for i, node := range nodes { log := t.streamClient.NewGetLogs(node). SetLines(&lines). SetFollow(&follow) - if !t.viewPath.IsZero() { - l := naming.Paths{t.viewPath}.StrSlice() + if !path.IsZero() { + l := naming.Paths{path}.StrSlice() log = log.SetPaths(&l) } reader, err := log.GetReader() @@ -1882,7 +1889,11 @@ func (t *App) updateLogTextView() { t.errorf("%s", err) continue } - t.logCloser.Append(reader) + if !t.logCloser.Append(reader) { + // the user left the log view while the readers were opening + _ = reader.Close() + return + } streams = append(streams, logreader.NodeStream{ Node: node, @@ -1891,16 +1902,17 @@ func (t *App) updateLogTextView() { }) } - if len(streams) > 0 { - // Use the logreader utility to collect, sort, and display logs - // Pass the TUI text view writer as the output - go logreader.CollectAndSortWithFormat( - streams, - outputWriter, // TUI text view writer - "", // output format - follow, // follow mode - ) + if len(streams) == 0 { + return } + // Use the logreader utility to collect, sort, and display logs. + // Pass the TUI text view writer as the output. + logreader.CollectAndSortWithFormat( + streams, + w, // TUI text view writer + "", // output format + follow, // follow mode + ) } func (t *App) getConfigUpdatedAt() time.Time { diff --git a/core/tui/view.go b/core/tui/view.go index f375260c7..3ef8d2c5c 100644 --- a/core/tui/view.go +++ b/core/tui/view.go @@ -144,6 +144,7 @@ func init() { title: "log", enter: func(t *App) { t.mountTextView() + t.logCloser.Reset() // updateLogTextView() opens the log readers and lets them stream // into the text view: it must not be called on data updates. t.updateLogTextView() From 46e01eace551ae9547abd8edb36369089315ec6d Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 09:01:48 +0200 Subject: [PATCH 41/54] Stop the tui events view from racing with the tview loop t.focus() reads the navigation stack, which nav() and back() write from the tview loop, and do() called it from its own goroutine to decide whether to refresh the object view clock. Mirror the focused view id in an atomic and read that off the loop. The events view goroutine read t.textView, which the view leave hook nils from the tview loop, guarding it with a plain nil check that happens to be a data race. Hand the text view and the context over to the goroutine, like the log view now does with its writer. t.stopEvents, toggled from the tview loop and read by that goroutine, becomes an atomic too. Also make the event fan out to the events view a non blocking send. The channel is only drained while that view is displayed, and the producer stops one check after the view is left: a stale event stays in the buffer at every visit, and once a hundred of them piled up the send blocked the whole event pipeline for good. An event dropped there only misses a line in the live tail, the cluster data is applied regardless. The remaining known race is the log and events views streaming into their tview TextView from a goroutine, concurrently with the draw loop. Fixing it means funneling those writes through QueueUpdateDraw. Until then the smoke test skips under the race detector. --- core/tui/events.go | 30 +++++++++++++----------------- core/tui/main.go | 17 +++++++++++++---- core/tui/race_off_test.go | 5 +++++ core/tui/race_on_test.go | 5 +++++ core/tui/smoke_test.go | 12 +++++++++++- core/tui/view.go | 10 +++++++++- 6 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 core/tui/race_off_test.go create mode 100644 core/tui/race_on_test.go diff --git a/core/tui/events.go b/core/tui/events.go index 94ac3b458..764bd7b08 100644 --- a/core/tui/events.go +++ b/core/tui/events.go @@ -19,7 +19,7 @@ func formatJSON(data json.RawMessage) string { func (t *App) getEventsViewTitle() string { state := "" - if t.stopEvents { + if t.stopEvents.Load() { state = "(paused)" } return fmt.Sprintf("events %s", state) @@ -30,7 +30,7 @@ func (t *App) initEventsView() { t.textView.Clear() t.textView.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { if event.Rune() == ' ' { - t.stopEvents = !t.stopEvents + t.stopEvents.Store(!t.stopEvents.Load()) t.textView.SetTitle(t.getEventsViewTitle()) } return event @@ -43,7 +43,6 @@ func (t *App) initEventsView() { } func (t *App) updateEventsView() { - if t.textView == nil { return } @@ -52,32 +51,29 @@ func (t *App) updateEventsView() { t.eventsCancel() } - t.eventsCtx, t.eventsCancel = context.WithCancel(context.Background()) + var ctx context.Context + ctx, t.eventsCancel = context.WithCancel(context.Background()) + + // Hand the text view and the context over to the goroutine: t.textView is + // nil'ed by the view leave hook, on the tview loop. + view := t.textView go func() { for { select { case event := <-t.events: - if t.stopEvents { + if t.stopEvents.Load() { continue } - - if t.textView == nil { - return - } - err := eventTemplate.Execute(t.textView, event) - - if err != nil { + if err := eventTemplate.Execute(view, event); err != nil { t.errorf("%s", err) return } - - fmt.Fprintln(t.textView) - t.textView.ScrollToEnd() - case <-t.eventsCtx.Done(): + fmt.Fprintln(view) + view.ScrollToEnd() + case <-ctx.Done(): return } } }() - } diff --git a/core/tui/main.go b/core/tui/main.go index f35dcef72..06b20497e 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -84,6 +84,10 @@ type ( stack viewStack + // focusedView mirrors the id of the focused stack frame, for the + // goroutines running off the tview loop. + focusedView atomic.Int32 + app *tview.Application top *tview.TextView head *tview.Table @@ -110,8 +114,7 @@ type ( mounted mountSpec events chan event.Event - stopEvents bool - eventsCtx context.Context + stopEvents atomic.Bool eventsCancel context.CancelFunc isInEventView atomic.Bool @@ -560,7 +563,13 @@ func (t *App) do(evReader event.ReadCloser) error { return err case e := <-eventC: if t.isInEventView.Load() { - t.events <- e + // Never block the event pipeline on the events view: it stops + // draining as soon as the user leaves it, and a blocked send + // here deadlocks the whole application. + select { + case t.events <- e: + default: + } } if nextEventID == 0 { nextEventID = e.ID @@ -581,7 +590,7 @@ func (t *App) do(evReader event.ReadCloser) error { if changes { dataC <- cdata.DeepCopy() changes = false - } else if t.focus() == viewObject { + } else if t.focusAsync() == viewObject { t.app.QueueUpdateDraw(func() { s := fmt.Sprint(time.Now().Truncate(time.Second).Sub(t.lastDraw.Truncate(time.Second))) t.objects.SetCell(2, 1, tview.NewTableCell(s).SetSelectable(false)) diff --git a/core/tui/race_off_test.go b/core/tui/race_off_test.go new file mode 100644 index 000000000..c1af6e696 --- /dev/null +++ b/core/tui/race_off_test.go @@ -0,0 +1,5 @@ +//go:build !race + +package tui + +const raceDetectorEnabled = false diff --git a/core/tui/race_on_test.go b/core/tui/race_on_test.go new file mode 100644 index 000000000..5bb30f00a --- /dev/null +++ b/core/tui/race_on_test.go @@ -0,0 +1,5 @@ +//go:build race + +package tui + +const raceDetectorEnabled = true diff --git a/core/tui/smoke_test.go b/core/tui/smoke_test.go index 1ca149996..de228b1c6 100644 --- a/core/tui/smoke_test.go +++ b/core/tui/smoke_test.go @@ -27,6 +27,14 @@ func TestSmokeLiveDaemon(t *testing.T) { t.Skipf("no daemon: %s", resp.Status()) } + if raceDetectorEnabled { + // The log and the events views stream into their tview TextView from a + // goroutine, racing with the tview draw loop. That predates the view + // registry, and fixing it means funneling those writes through + // QueueUpdateDraw. + t.Skip("the log and events views race with the tview draw loop") + } + screen := tcell.NewSimulationScreen("UTF-8") if err := screen.Init(); err != nil { t.Fatal(err) @@ -67,7 +75,6 @@ func TestSmokeLiveDaemon(t *testing.T) { // let the first cluster data land time.Sleep(2 * time.Second) - t.Logf("nodes=%d objects=%d", len(a.Current.Cluster.Config.Nodes), len(a.Current.Cluster.Object)) var dump func() string sync := func(f func()) { @@ -86,6 +93,9 @@ func TestSmokeLiveDaemon(t *testing.T) { sync(func() { s = readScreen() }) return s } + sync(func() { + t.Logf("nodes=%d objects=%d", len(a.Current.Cluster.Config.Nodes), len(a.Current.Cluster.Object)) + }) key := func(k tcell.Key) { screen.InjectKey(k, 0, tcell.ModNone) sync(func() {}) diff --git a/core/tui/view.go b/core/tui/view.go index 3ef8d2c5c..a866a9fb8 100644 --- a/core/tui/view.go +++ b/core/tui/view.go @@ -229,11 +229,18 @@ func (t *App) frame() *frame { return &t.stack[len(t.stack)-1] } -// focus returns the id of the focused view. +// focus returns the id of the focused view. The navigation stack is owned by +// the tview loop: call focusAsync() from any other goroutine. func (t *App) focus() viewId { return t.frame().id } +// focusAsync returns the id of the focused view, read from the atomic mirror +// enterView() keeps up to date. Safe to call off the tview loop. +func (t *App) focusAsync() viewId { + return viewId(t.focusedView.Load()) +} + // atRoot returns true when the focused view is the one ESC can not pop. func (t *App) atRoot() bool { return len(t.stack) == 1 @@ -285,6 +292,7 @@ func (t *App) back() { } func (t *App) enterView(v viewId) { + t.focusedView.Store(int32(v)) t.lastUpdatedAt = time.Time{} if enter := viewDefs[v].enter; enter != nil { enter(t) From 5bcdd2ecf7e88f5945a0d84270205c2c1aae6346 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 09:44:06 +0200 Subject: [PATCH 42/54] Stop the tui log and events views from racing with the tview loop The log and the events views are written into by the goroutines streaming their content. TextView.Write() takes the text view lock, so that part is fine, but both views also called TextView.ScrollToEnd() off the tview loop, and that one writes trackEnd and columnOffset with no lock at all, concurrently with TextView.Draw() reading them. The log view called it from the changed handler, which tview invokes in a goroutine of its own, and the events view called it straight from its streaming goroutine. Set the follow flag once instead, on the tview loop, when the view is entered: nothing clears it until the user scrolls up, so following the tail does not need a call per line. That leaves the changed handlers with the one thing tview documents as allowed there, Application.Draw(), which neither view was doing: writing into a text view does not refresh the screen, and the tui only redrew on a cluster data change. Both tails froze on an idle cluster. They now repaint as the lines land. Also stop clearing the changed handler when leaving the log view: SetChangedFunc() is an unlocked field write too, and the readers are still reading it while they wind down. The handler only asks for a redraw, so it costs nothing on a text view left behind. TestStreamedTextViewRedraws covers both views, and the smoke test no longer needs to skip under the race detector. It reads the application state from the tview loop, which owns it. --- core/tui/events.go | 7 ++- core/tui/main.go | 14 +++-- core/tui/race_off_test.go | 5 -- core/tui/race_on_test.go | 5 -- core/tui/smoke_test.go | 86 ++++++++++++++++++++---------- core/tui/textview_test.go | 108 ++++++++++++++++++++++++++++++++++++++ core/tui/view.go | 7 +-- 7 files changed, 188 insertions(+), 44 deletions(-) delete mode 100644 core/tui/race_off_test.go delete mode 100644 core/tui/race_on_test.go create mode 100644 core/tui/textview_test.go diff --git a/core/tui/events.go b/core/tui/events.go index 764bd7b08..ded44ff74 100644 --- a/core/tui/events.go +++ b/core/tui/events.go @@ -28,6 +28,12 @@ func (t *App) getEventsViewTitle() string { func (t *App) initEventsView() { t.textView.SetTitle(t.getEventsViewTitle()) t.textView.Clear() + + // Follow the tail, and let the writes refresh the screen. Both have to be + // set here, on the tview loop: see updateLogTextView(). + t.textView.ScrollToEnd() + t.textView.SetChangedFunc(func() { t.app.Draw() }) + t.textView.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { if event.Rune() == ' ' { t.stopEvents.Store(!t.stopEvents.Load()) @@ -70,7 +76,6 @@ func (t *App) updateEventsView() { return } fmt.Fprintln(view) - view.ScrollToEnd() case <-ctx.Done(): return } diff --git a/core/tui/main.go b/core/tui/main.go index 06b20497e..7df0da090 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -1847,11 +1847,19 @@ func (t *App) updateLogTextView() { t.textView.SetTitle(title()) t.textView.SetDynamicColors(true) - t.textView.SetChangedFunc(func() { - t.textView.ScrollToEnd() - }) t.textView.Clear() + // Follow the tail. TextView.ScrollToEnd() writes unlocked fields, so it + // belongs here, on the tview loop, not in the changed handler which tview + // calls from a goroutine of its own. Once set, the flag survives the + // writes: only the user scrolling up clears it. + t.textView.ScrollToEnd() + + // The log readers write into the text view from their own goroutine, and + // writing does not refresh the screen. Application.Draw() is the one call + // tview allows from a changed handler. + t.textView.SetChangedFunc(func() { t.app.Draw() }) + var nodes []string if t.viewNode != "" { // instance or node logs diff --git a/core/tui/race_off_test.go b/core/tui/race_off_test.go deleted file mode 100644 index c1af6e696..000000000 --- a/core/tui/race_off_test.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build !race - -package tui - -const raceDetectorEnabled = false diff --git a/core/tui/race_on_test.go b/core/tui/race_on_test.go deleted file mode 100644 index 5bb30f00a..000000000 --- a/core/tui/race_on_test.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build race - -package tui - -const raceDetectorEnabled = true diff --git a/core/tui/smoke_test.go b/core/tui/smoke_test.go index de228b1c6..55602831c 100644 --- a/core/tui/smoke_test.go +++ b/core/tui/smoke_test.go @@ -27,14 +27,6 @@ func TestSmokeLiveDaemon(t *testing.T) { t.Skipf("no daemon: %s", resp.Status()) } - if raceDetectorEnabled { - // The log and the events views stream into their tview TextView from a - // goroutine, racing with the tview draw loop. That predates the view - // registry, and fixing it means funneling those writes through - // QueueUpdateDraw. - t.Skip("the log and events views race with the tview draw loop") - } - screen := tcell.NewSimulationScreen("UTF-8") if err := screen.Init(); err != nil { t.Fatal(err) @@ -93,6 +85,28 @@ func TestSmokeLiveDaemon(t *testing.T) { sync(func() { s = readScreen() }) return s } + // state is a snapshot of the application state, read on the tview loop: + // nav() and the table cursor callbacks own it from there. + type state struct { + focus viewId + stack string + depth int + element string + flexItems int + } + inspect := func() state { + var st state + sync(func() { + st = state{ + focus: a.focus(), + stack: a.stack.String(), + depth: len(a.stack), + element: a.selectedElement, + flexItems: a.flex.GetItemCount(), + } + }) + return st + } sync(func() { t.Logf("nodes=%d objects=%d", len(a.Current.Cluster.Config.Nodes), len(a.Current.Cluster.Object)) }) @@ -104,55 +118,73 @@ func TestSmokeLiveDaemon(t *testing.T) { for _, v := range []viewId{viewHbStatus, viewPool, viewNetwork, viewRelay, viewEvents, viewConfig, viewLog} { sync(func() { a.nav(v) }) time.Sleep(300 * time.Millisecond) - if got := a.focus(); got != v { - t.Fatalf("nav to %s: focused on %s", v, got) + if st := inspect(); st.focus != v { + t.Fatalf("nav to %s: focused on %s", v, st.focus) } for i := 0; i < 3; i++ { key(tcell.KeyPgDn) } key(tcell.KeyPgUp) key(tcell.KeyDown) - if n := a.flex.GetItemCount(); n < 3 { - t.Errorf("%s: only %d flex items", v, n) + + st := inspect() + if st.flexItems < 3 { + t.Errorf("%s: only %d flex items", v, st.flexItems) } head := strings.TrimSpace(strings.SplitN(dump(), "\n", 2)[0]) - t.Logf("%-18s head=%q stack=%s", v, head, a.stack) + t.Logf("%-18s head=%q stack=%s", v, head, st.stack) if head == "" { t.Errorf("%s: the head bar is empty", v) } + sync(func() { a.back() }) time.Sleep(200 * time.Millisecond) - if !a.atRoot() || a.focus() != viewObject { - t.Fatalf("back from %s: stack is %s", v, a.stack) + if st := inspect(); st.depth != 1 || st.focus != viewObject { + t.Fatalf("back from %s: stack is %s", v, st.stack) } } + // drill down into the first pool, and back out sync(func() { a.nav(viewPool) }) time.Sleep(300 * time.Millisecond) - table, ok := a.body().(*tview.Table) - if !ok { - t.Fatalf("the pool view body is a %T", a.body()) + var ( + poolName string + rows int + isTable bool + ) + sync(func() { + table, ok := a.body().(*tview.Table) + if !ok { + return + } + isTable = true + if rows = table.GetRowCount(); rows < 2 { + return + } + table.Select(1, 0) + poolName = table.GetCell(1, 0).Text + }) + if !isTable { + t.Fatal("the pool view body is not a table") } - if table.GetRowCount() < 2 { + if rows < 2 { t.Skip("no pool to drill down into") } - sync(func() { table.Select(1, 0) }) - poolName := table.GetCell(1, 0).Text key(tcell.KeyEnter) time.Sleep(300 * time.Millisecond) - if a.focus() != viewPool || len(a.stack) != 3 || a.selectedElement != poolName { - t.Fatalf("drilling into pool %q: stack=%s element=%q", poolName, a.stack, a.selectedElement) + if st := inspect(); st.focus != viewPool || st.depth != 3 || st.element != poolName { + t.Fatalf("drilling into pool %q: stack=%s element=%q", poolName, st.stack, st.element) } t.Logf("drilled into pool %q, head=%q", poolName, strings.TrimSpace(strings.SplitN(dump(), "\n", 2)[0])) sync(func() { a.back() }) time.Sleep(300 * time.Millisecond) - if a.focus() != viewPool || len(a.stack) != 2 || a.selectedElement != "" { - t.Fatalf("back to the pool list: stack=%s element=%q", a.stack, a.selectedElement) + if st := inspect(); st.focus != viewPool || st.depth != 2 || st.element != "" { + t.Fatalf("back to the pool list: stack=%s element=%q", st.stack, st.element) } sync(func() { a.back() }) - if !a.atRoot() { - t.Fatalf("back to the object view: stack=%s", a.stack) + if st := inspect(); st.depth != 1 { + t.Fatalf("back to the object view: stack=%s", st.stack) } t.Logf("final screen:\n%s", dump()) diff --git a/core/tui/textview_test.go b/core/tui/textview_test.go new file mode 100644 index 000000000..5e5f2c627 --- /dev/null +++ b/core/tui/textview_test.go @@ -0,0 +1,108 @@ +package tui + +import ( + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +// countingScreen counts the screen refreshes. +type countingScreen struct { + tcell.SimulationScreen + shows atomic.Int64 +} + +func (s *countingScreen) Show() { + s.shows.Add(1) + s.SimulationScreen.Show() +} + +// The log and events views are written into by the goroutines streaming their +// content, which have no way to refresh the screen themselves: the text view +// changed handler has to do it for them. Run with -race: the writes must not +// race with the draw loop either. +func TestStreamedTextViewRedraws(t *testing.T) { + for _, test := range []struct { + view viewId + name string + }{ + {viewLog, "log"}, + {viewEvents, "events"}, + } { + t.Run(test.name, func(t *testing.T) { + sim := tcell.NewSimulationScreen("UTF-8") + if err := sim.Init(); err != nil { + t.Fatal(err) + } + sim.SetSize(120, 10) + screen := &countingScreen{SimulationScreen: sim} + + a := NewApp(nil) + a.initHeadTextView() + a.initObjectsTable() + a.initErrsTextView() + a.app = tview.NewApplication().SetScreen(screen) + a.flex = tview.NewFlex().SetDirection(tview.FlexRow) + a.app.SetRoot(a.flex, true) + a.mount(a.objects) + + done := make(chan struct{}) + go func() { defer close(done); _ = a.app.Run() }() + defer func() { + a.app.Stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("the application did not stop") + } + }() + + sync := func(f func()) { + ch := make(chan struct{}) + a.app.QueueUpdateDraw(func() { f(); close(ch) }) + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("the event loop is wedged") + } + } + + // no cluster data: the views mount their text view and stream + // nothing, which is all this test needs + var view *tview.TextView + sync(func() { + a.nav(test.view) + view = a.textView + }) + if view == nil { + t.Fatal("the view mounted no text view") + } + + before := screen.shows.Load() + + // stand in for the streaming goroutine + written := make(chan struct{}) + go func() { + defer close(written) + for i := 0; i < 20; i++ { + fmt.Fprintf(view, "line %d\n", i) + } + }() + <-written + + deadline := time.Now().Add(5 * time.Second) + for screen.shows.Load() == before && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if got := screen.shows.Load(); got == before { + t.Errorf("writing into the %s view refreshed no screen (%d shows)", test.name, got) + } else { + t.Logf("%s: %d screen refreshes for 20 written lines", test.name, got-before) + } + }) + } +} diff --git a/core/tui/view.go b/core/tui/view.go index a866a9fb8..ff0b55832 100644 --- a/core/tui/view.go +++ b/core/tui/view.go @@ -150,9 +150,10 @@ func init() { t.updateLogTextView() }, leave: func(t *App) { - if t.textView != nil { - t.textView.SetChangedFunc(nil) - } + // Don't clear the changed handler: TextView.SetChangedFunc() + // writes an unlocked field, which the log readers are reading + // from their own goroutine. The handler only asks for a + // redraw, so it is harmless on a text view left behind. t.releaseTextView() t.logCloser.CloseAll() }, From 94c96ea9fe0d80021fc7a31e9cb9c91f88b05de1 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 10:10:00 +0200 Subject: [PATCH 43/54] Stop the tui errors bar from racing with the tview loop printf() wrote the errors bar wherever it was called from, and errorf() is called from the goroutines nobody thinks about: the event reader, the event to message conversion, the log readers and the events view. Two of its writes are tview.Box.SetBackgroundColor(), an unlocked field write, concurrent with the draw loop reading it. The second one came from a time.AfterFunc, so it was off the tview loop even for the callers that were on it. Give the bar a single writer: printf() now queues the message and runErrsBar() displays it, from the tview loop, through QueueUpdateDraw. The queueing is a non blocking send, so the goroutines feeding the event pipeline can no longer be held up by the display. Dropping a message the bar has no room for is what the user got anyway: each printf() used to schedule its own expiry timer, and an early one cleared a later message. A single timer, reset on each message, gives every message its full lingering time. --- core/tui/main.go | 69 ++++++++++++++++++++++++++++++++------ core/tui/textview_test.go | 70 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 10 deletions(-) diff --git a/core/tui/main.go b/core/tui/main.go index 7df0da090..732c9f0f4 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -50,6 +50,12 @@ type ( defaultText string } + // errsMessage is a message queued for display in the errors bar. + errsMessage struct { + color tcell.Color + text string + } + CreateTableOptions struct { title string titles []string @@ -144,9 +150,15 @@ type ( selectedInstances map[[2]string]any selectedRIDs map[[3]string]any - errC chan error - restartC chan error - exitFlag atomic.Bool + // errsC carries the messages printf() queues for the errors bar, and + // errsLinger is how long each of them stays displayed. + errsC chan errsMessage + errsLinger time.Duration + errC chan error + restartC chan error + stopC chan struct{} + stopOnce sync.Once + exitFlag atomic.Bool logCloser AtomicCloserSlice } @@ -228,6 +240,9 @@ func NewApp(options *Options) *App { selectedRIDs: make(map[[3]string]any), errC: make(chan error), restartC: make(chan error), + stopC: make(chan struct{}), + errsC: make(chan errsMessage, 8), + errsLinger: 5 * time.Second, events: make(chan event.Event, 100), } } @@ -272,6 +287,8 @@ func (t *App) initApp() { t.app.SetRoot(t.flex, true) t.mount(t.objects) + go t.runErrsBar() + t.app.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { if t.command != nil { return event @@ -2231,14 +2248,45 @@ func (t *App) errorf(format string, args ...any) { t.printf(tcell.ColorRed, format, args...) } +// printf queues a message for the errors bar. Safe to call from any +// goroutine: the bar is owned by runErrsBar(), the only writer, and it only +// writes from the tview loop. +// +// The send never blocks, so the goroutines feeding the event pipeline are +// never held up by the display. A message dropped because the bar is +// congested is a message nobody had the time to read anyway. func (t *App) printf(color tcell.Color, format string, args ...any) { - t.errs.Clear() - t.errs.SetBackgroundColor(color) - fmt.Fprintf(t.errs, format, args...) - time.AfterFunc(5*time.Second, func() { - t.errs.Clear() - t.errs.SetBackgroundColor(colorNone) - }) + select { + case t.errsC <- errsMessage{color: color, text: fmt.Sprintf(format, args...)}: + default: + } +} + +// runErrsBar displays the messages printf() queues, each of them lingering +// errsLinger before the bar goes back to empty. It owns the errors bar +// primitive: tview.Box.SetBackgroundColor() writes an unlocked field, so it +// has to run on the tview loop, concurrently with nothing. +func (t *App) runErrsBar() { + show := func(color tcell.Color, text string) { + t.app.QueueUpdateDraw(func() { + t.errs.SetBackgroundColor(color) + t.errs.Clear() + fmt.Fprint(t.errs, text) + }) + } + var lingerC <-chan time.Time + for { + select { + case <-t.stopC: + return + case m := <-t.errsC: + show(m.color, m.text) + lingerC = time.After(t.errsLinger) + case <-lingerC: + lingerC = nil + show(colorNone, "") + } + } } func (t *App) createTable(creator CreateTableOptions) { @@ -2355,6 +2403,7 @@ func (t *App) reconnect() { func (t *App) stop() { t.exitFlag.Store(true) + t.stopOnce.Do(func() { close(t.stopC) }) select { case t.errC <- nil: default: diff --git a/core/tui/textview_test.go b/core/tui/textview_test.go index 5e5f2c627..f8b9dd327 100644 --- a/core/tui/textview_test.go +++ b/core/tui/textview_test.go @@ -2,6 +2,7 @@ package tui import ( "fmt" + "strings" "sync/atomic" "testing" "time" @@ -106,3 +107,72 @@ func TestStreamedTextViewRedraws(t *testing.T) { }) } } + +// The errors bar is written into from any goroutine, tview.Box's background +// color being an unlocked field: only runErrsBar() may touch it, and only from +// the tview loop. Run with -race. +func TestErrorBarWritesFromAnyGoroutine(t *testing.T) { + sim := tcell.NewSimulationScreen("UTF-8") + if err := sim.Init(); err != nil { + t.Fatal(err) + } + sim.SetSize(120, 10) + + a := NewApp(nil) + a.errsLinger = 300 * time.Millisecond + a.initHeadTextView() + a.initObjectsTable() + a.initErrsTextView() + a.app = tview.NewApplication().SetScreen(sim) + a.flex = tview.NewFlex().SetDirection(tview.FlexRow) + a.app.SetRoot(a.flex, true) + a.mount(a.objects) + go a.runErrsBar() + + done := make(chan struct{}) + go func() { defer close(done); _ = a.app.Run() }() + + barText := func() string { + var s string + ch := make(chan struct{}) + a.app.QueueUpdateDraw(func() { s = a.errs.GetText(true); close(ch) }) + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("the event loop is wedged") + } + return s + } + await := func(what string, ok func(string) bool) string { + deadline := time.Now().Add(5 * time.Second) + for { + s := barText() + if ok(s) { + return s + } + if time.Now().After(deadline) { + t.Fatalf("%s: the errors bar holds %q", what, s) + } + time.Sleep(20 * time.Millisecond) + } + } + + // off the tview loop, as do() and the streaming goroutines do + go a.errorf("boom %d", 42) + + got := await("the message never showed", func(s string) bool { + return strings.Contains(s, "boom 42") + }) + t.Logf("errors bar: %q", strings.TrimSpace(got)) + + await("the message never expired", func(s string) bool { + return strings.TrimSpace(s) == "" + }) + + a.stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("the application did not stop") + } +} From 0f8b9a0618a7454d499dd1bbf0c24bea1fe71f98 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 10:10:07 +0200 Subject: [PATCH 44/54] Guard the sse ReadCloser closed flag against concurrent Read and Close ReadCloser.closed was a plain bool, written by Close() and read by Read(). Those two run on different goroutines whenever a stream is closed by someone other than its reader, which is what the tui does to the log readers when the user leaves the log view. Make it an atomic, and let Close() claim it with a swap so a concurrent second Close still gets ErrClosed. --- core/event/sseevent/main.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/event/sseevent/main.go b/core/event/sseevent/main.go index 13c933a46..2121b7e02 100644 --- a/core/event/sseevent/main.go +++ b/core/event/sseevent/main.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "strconv" + "sync/atomic" "time" "github.com/opensvc/om3/v3/core/event" @@ -27,8 +28,10 @@ type ( // parseStarted become true during first Read (internal go routine is parseStarted) parseStarted bool - // closed is true when Close() is called - closed bool + // closed is true when Close() is called. Read() and Close() are + // called from different goroutines: the log and event readers are + // closed by the reader of their stream. + closed atomic.Bool // max is the maxTokenSize for internal scanner max int @@ -72,7 +75,6 @@ func NewReadCloser(r io.ReadCloser) *ReadCloser { errC: make(chan error), wrapped: r, parseStarted: false, - closed: false, max: MaxScanTokenSize, buf: make([]byte, initialBufferSize), } @@ -119,7 +121,7 @@ func (r *ReadCloser) Buffer(buf []byte, max int) { // Read returns *Event read from EventReader r func (r *ReadCloser) Read() (*event.Event, error) { - if r.closed { + if r.closed.Load() { return nil, ErrClosed } if !r.parseStarted { @@ -160,11 +162,10 @@ func (r *ReadCloser) Read() (*event.Event, error) { // Close ask wrapped io.readCloser for Close func (r *ReadCloser) Close() error { - if r.closed { + if r.closed.Swap(true) { return ErrClosed } r.cancel() - r.closed = true return r.wrapped.Close() } From 9b0f77c794a89250e6da40a67cdf386c97f19680 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 10:10:18 +0200 Subject: [PATCH 45/54] Publish the tui cluster snapshot from the tview loop t.Current, t.Nodename and t.eventCount were assigned by the goroutine draining the cluster data, then the views were told to repaint through QueueUpdateDraw. That orders the assignment against the repaint, but not against the key handlers, which read the same snapshot from the tview loop: navigating to a view while new data lands has the view build its table from a snapshot being overwritten under it. Assign inside the queued function, so the snapshot is written and read from the tview loop only. --- core/tui/main.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/tui/main.go b/core/tui/main.go index 732c9f0f4..fc601f6e1 100644 --- a/core/tui/main.go +++ b/core/tui/main.go @@ -549,18 +549,21 @@ func (t *App) do(evReader event.ReadCloser) error { wg.Add(1) go func(d *clusterdump.Data) { defer wg.Done() - t.Current = *d - t.Nodename = data.Daemon.Nodename + // The cluster snapshot the views read is published from the tview + // loop: the views also read it from the key handlers, which run + // there, with no ordering against this goroutine. t.app.QueueUpdateDraw(func() { + t.Current = *d + t.Nodename = data.Daemon.Nodename t.updateHead() t.updateObjects() }) // show data when new data published on dataC for d := range dataC { - t.Current = *d - t.Nodename = data.Daemon.Nodename - t.eventCount++ t.app.QueueUpdateDraw(func() { + t.Current = *d + t.Nodename = data.Daemon.Nodename + t.eventCount++ // TODO: detect if t.updateInstanceView and t.updateConfigView need to be called (config mtime change, ...) t.refreshView() }) From 2cf374de7919d5d3c125acc97ca8a4058bd0c835 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 10:27:53 +0200 Subject: [PATCH 46/54] Show the resource run in progress in the instance cell The daemon publishes one instance.status.running entry per resource run in progress, a task or a sync, but nothing surfaced it in the daemon status renderer: users had to open the instance status of every instance to find out where a run was going on. Add an R flag to the instance cell of the objects section, right after the availability and warning icons, so it shows in the om and ox monitor output and in the ox tui main page, which share the renderer. Count the encapsulated instances runs too. Their run info lives in the encap status, which nothing reads: ResourceFlagsString() looks the encap rids up in the running list of the outer instance, where they are not. --- core/monitor/frame.go | 2 + core/monitor/frame_instance.go | 16 ++++++ core/monitor/frame_instance_test.go | 49 +++++++++++++++++++ core/monitor/main.go | 1 + .../testdata/multi-node-daemon-status.json | 16 ++++++ .../testdata/multi-node-om-mon.fixture | 2 +- 6 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 core/monitor/frame_instance_test.go diff --git a/core/monitor/frame.go b/core/monitor/frame.go index d9c960a31..97b35dc13 100644 --- a/core/monitor/frame.go +++ b/core/monitor/frame.go @@ -38,6 +38,7 @@ var ( iconProvisionAlert, iconStandbyDown, iconStandbyUpIssue string iconUndef, iconFrozen, iconDown, iconDRP, iconLeader string iconNotApplicable, iconPreserved, iconStandbyUp string + iconRunning string now = time.Now ) @@ -65,6 +66,7 @@ func InitColor() { iconNotApplicable = hiBlack("/") iconPreserved = hiBlack("?") iconStandbyUp = hiBlack("o") + iconRunning = hiBlue("R") } type ( diff --git a/core/monitor/frame_instance.go b/core/monitor/frame_instance.go index e16422ed5..0380f281d 100644 --- a/core/monitor/frame_instance.go +++ b/core/monitor/frame_instance.go @@ -34,6 +34,7 @@ func (f Frame) StrObjectInstance(path string, node string, scope []string) strin instanceStatus := *inst.Status s += sObjectInstanceAvail(avail, instanceStatus, instanceMonitor) s += sObjectInstanceOverall(instanceStatus) + s += sObjectInstanceRunning(instanceStatus) s += sObjectInstanceDRP(instanceConfig) s += sObjectInstanceHALeader(instanceMonitor) s += sObjectInstanceFrozen(instanceStatus) @@ -83,6 +84,21 @@ func sObjectInstanceOverall(instance instance.Status) string { return "" } +// sObjectInstanceRunning marks the instances having at least one resource run +// in progress: a task or a sync. The daemon feeds instance.Status.Running from +// the resource run files, one entry per running resource. +func sObjectInstanceRunning(instance instance.Status) string { + if len(instance.Running) > 0 { + return iconRunning + } + for _, encap := range instance.Encap { + if len(encap.Running) > 0 { + return iconRunning + } + } + return "" +} + func sObjectInstanceDRP(instance instance.Config) string { if instance.ActorConfig != nil && instance.ActorConfig.DRP { return iconDRP diff --git a/core/monitor/frame_instance_test.go b/core/monitor/frame_instance_test.go new file mode 100644 index 000000000..fb5e8601f --- /dev/null +++ b/core/monitor/frame_instance_test.go @@ -0,0 +1,49 @@ +package monitor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/opensvc/om3/v3/core/instance" + "github.com/opensvc/om3/v3/core/resource" +) + +func TestObjectInstanceRunning(t *testing.T) { + InitColor() + running := resource.RunningInfoList{{RID: "task#3", PID: 2265402}} + + for name, test := range map[string]struct { + status instance.Status + expected string + }{ + "nothing running": { + status: instance.Status{}, + }, + "a resource of the instance is running": { + status: instance.Status{Running: running}, + expected: iconRunning, + }, + "a resource of an encapsulated instance is running": { + status: instance.Status{ + Encap: instance.EncapMap{ + "container#1": instance.EncapStatus{ + Status: instance.Status{Running: running}, + }, + }, + }, + expected: iconRunning, + }, + "no resource of the encapsulated instance is running": { + status: instance.Status{ + Encap: instance.EncapMap{ + "container#1": instance.EncapStatus{}, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.expected, sObjectInstanceRunning(test.status)) + }) + } +} diff --git a/core/monitor/main.go b/core/monitor/main.go index b4e5e5e2a..4c0dc82b4 100644 --- a/core/monitor/main.go +++ b/core/monitor/main.go @@ -51,6 +51,7 @@ Instance Flags: * Frozen ^ Placement leader # DRP instance + R Resource run in progress ` // New allocates a monitor. diff --git a/core/monitor/testdata/multi-node-daemon-status.json b/core/monitor/testdata/multi-node-daemon-status.json index b23f72b2a..4a4442a03 100644 --- a/core/monitor/testdata/multi-node-daemon-status.json +++ b/core/monitor/testdata/multi-node-daemon-status.json @@ -247,6 +247,14 @@ "last_started_at": "0001-01-01T00:00:00Z", "overall": "up", "provisioned": "true", + "running": [ + { + "at": "2026-08-28T10:16:28.053350726+02:00", + "pid": 2265402, + "rid": "task#3", + "session_id": "0f94b866-97b0-4504-acc6-88d9ebe15b0c" + } + ], "resources": { "fs#1": { "label": "flag /dev/shm/opensvc/svc/foo/fs#1.flag", @@ -718,6 +726,14 @@ "last_started_at": "0001-01-01T00:00:00Z", "overall": "down", "provisioned": "true", + "running": [ + { + "at": "2026-08-28T10:16:28.053350726+02:00", + "pid": 2265402, + "rid": "task#3", + "session_id": "0f94b866-97b0-4504-acc6-88d9ebe15b0c" + } + ], "resources": { "fs#1": { "label": "flag /dev/shm/opensvc/svc/foo/fs#1.flag", diff --git a/core/monitor/testdata/multi-node-om-mon.fixture b/core/monitor/testdata/multi-node-om-mon.fixture index de068550e..02f1ba703 100644 --- a/core/monitor/testdata/multi-node-om-mon.fixture +++ b/core/monitor/testdata/multi-node-om-mon.fixture @@ -15,6 +15,6 @@ Nodes node1 node2 node3 Objects ~ * node1 node2 node3 cluster n/a | / / / - foo up ha 1/1 | O^ X X + foo up ha 1/1 | OR^ XR X system/sec/ca n/a no | / / / system/sec/cert n/a no | / / / From 66bcd7b7c5b3f67f7a5371806b3b13a055daebe2 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 11:45:03 +0200 Subject: [PATCH 47/54] Exit non zero when the om and ox args name no command cobra walks the command chain and, when the deepest command it reaches has no Run function, prints that command help and returns no error, silently dropping whatever was typed after it. It does so before validating the leftover args, so there is no hook to reject them. A stale or mistyped command path exits 0 that way. That is how the daemon, which execs om with hard-coded argv from its scheduler and from its api action handlers, kept reporting successful runs of commands that had been renamed under it. Resolve the args before handing them to cobra, and refuse the ones leaving a command name behind a command that can not run. The leftover args have to be told from the flags to find that name, and cobra parses the same args again right after, so the parse is done on a throwaway flag set holding copies of the flag shapes, values discarded: parsing the real ones twice would append twice to the slice flags. setExecuteArgs() now returns the args it hands to cobra. Its rewriting of an object selector into a subsystem command is untouched. --- core/commoncmd/args.go | 91 ++++++++++++++++++++++++++++++++ core/commoncmd/args_test.go | 101 ++++++++++++++++++++++++++++++++++++ core/om/root.go | 12 +++-- core/ox/root.go | 10 +++- 4 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 core/commoncmd/args.go create mode 100644 core/commoncmd/args_test.go diff --git a/core/commoncmd/args.go b/core/commoncmd/args.go new file mode 100644 index 000000000..22a399551 --- /dev/null +++ b/core/commoncmd/args.go @@ -0,0 +1,91 @@ +package commoncmd + +import ( + "fmt" + "io" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// discardValue stands for a real flag value in the throwaway flag set +// firstNonFlag() parses with. Parsing must not write the values cobra is +// about to parse itself: a repeated parse would append twice to the slice +// flags. +type discardValue struct { + typ string +} + +func (t discardValue) String() string { return "" } +func (t discardValue) Set(string) error { return nil } +func (t discardValue) Type() string { return t.typ } + +// ValidateArgs returns an error when args name a command that does not exist. +// +// cobra walks the command chain and, when the deepest command it reaches has +// no Run function, prints that command help and returns no error, silently +// dropping whatever was typed after it. A stale or mistyped command path +// exits 0 that way, which is how the daemon scheduler and the api exec kept +// reporting successful runs of commands renamed under their hard-coded argv. +func ValidateArgs(root *cobra.Command, args []string) error { + if len(args) == 0 { + return nil + } + if strings.HasPrefix(args[0], "__complete") { + // shell completion: cobra answers with candidates, it does not run + return nil + } + cmd, rest, err := root.Find(args) + if err != nil || cmd == nil || cmd.Runnable() { + // an unknown first word is reported by cobra itself + return nil + } + name, ok := firstNonFlag(cmd, rest) + if !ok { + // no command name left: the user asked for the command list + return nil + } + return fmt.Errorf("unknown command %q for %q", name, cmd.CommandPath()) +} + +// firstNonFlag returns the first positional argument left in args once the +// flags cmd accepts are stripped. +func firstNonFlag(cmd *cobra.Command, args []string) (string, bool) { + fs := pflag.NewFlagSet(cmd.Name(), pflag.ContinueOnError) + fs.ParseErrorsWhitelist.UnknownFlags = true + fs.SetOutput(io.Discard) + + add := func(f *pflag.Flag) { + if fs.Lookup(f.Name) != nil { + return + } + shorthand := f.Shorthand + if shorthand != "" && fs.ShorthandLookup(shorthand) != nil { + shorthand = "" + } + fs.AddFlag(&pflag.Flag{ + Name: f.Name, + Shorthand: shorthand, + Value: discardValue{typ: f.Value.Type()}, + DefValue: f.DefValue, + // carried over so the valueless flags don't eat the next word + NoOptDefVal: f.NoOptDefVal, + }) + } + // InheritedFlags() and LocalFlags() both merge the parents persistent + // flags into cmd.Flags() on the way, which Flags() alone does not do + // before the command runs. + cmd.InheritedFlags().VisitAll(add) + cmd.LocalFlags().VisitAll(add) + cmd.Flags().VisitAll(add) + + if err := fs.Parse(args); err != nil { + // a flag error is cobra's to report + return "", false + } + if rest := fs.Args(); len(rest) > 0 { + return rest[0], true + } + return "", false +} diff --git a/core/commoncmd/args_test.go b/core/commoncmd/args_test.go new file mode 100644 index 000000000..b8a7bf4bc --- /dev/null +++ b/core/commoncmd/args_test.go @@ -0,0 +1,101 @@ +package commoncmd + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestTree returns a command tree shaped like the om one: a non runnable +// root, non runnable subsystem levels, and a runnable leaf. +func newTestTree() (*cobra.Command, *string) { + var selector string + root := &cobra.Command{Use: "om"} + root.PersistentFlags().StringVar(new(string), "color", "auto", "") + root.PersistentFlags().StringVarP(&selector, "selector", "s", "", "") + root.PersistentFlags().Bool("debug", false, "") + + svc := &cobra.Command{Use: "svc"} + instance := &cobra.Command{Use: "instance"} + status := &cobra.Command{ + Use: "status", + RunE: func(*cobra.Command, []string) error { + return nil + }, + } + status.Flags().BoolP("refresh", "r", false, "") + + instance.AddCommand(status) + svc.AddCommand(instance) + root.AddCommand(svc) + return root, &selector +} + +func TestValidateArgs(t *testing.T) { + for name, test := range map[string]struct { + args []string + unknown string + }{ + "a runnable leaf": { + args: []string{"svc", "instance", "status"}, + }, + "a runnable leaf and its flags": { + args: []string{"svc", "instance", "status", "-r"}, + }, + "a bare subsystem asks for the command list": { + args: []string{"svc", "instance"}, + }, + "no argument at all": { + args: nil, + }, + "a valueless flag does not eat the next word": { + args: []string{"svc", "instance", "--debug"}, + }, + "an unknown command under a subsystem": { + args: []string{"svc", "instance", "resource", "info", "push"}, + unknown: "resource", + }, + "a flag value is not mistaken for a command": { + args: []string{"svc", "-s", "test/svc/s1", "instance", "resource", "info", "push"}, + unknown: "resource", + }, + "a flag value passed with an equal sign": { + args: []string{"svc", "--color=no", "instance", "bogus"}, + unknown: "bogus", + }, + "a valueless flag before an unknown command": { + args: []string{"svc", "instance", "--debug", "bogus"}, + unknown: "bogus", + }, + "an unknown first word is left to cobra": { + args: []string{"nosuchsubsystem", "instance"}, + }, + "shell completion never runs a command": { + args: []string{"__complete", "svc", "instance", "bogus"}, + }, + } { + t.Run(name, func(t *testing.T) { + root, _ := newTestTree() + err := ValidateArgs(root, test.args) + if test.unknown == "" { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), `"`+test.unknown+`"`) + }) + } +} + +// ValidateArgs parses the args to tell the flags from the command names, and +// cobra parses them again right after: it must not have written anything. +func TestValidateArgsLeavesTheFlagValuesAlone(t *testing.T) { + root, selector := newTestTree() + args := []string{"svc", "-s", "test/svc/s1", "instance", "bogus"} + + require.Error(t, ValidateArgs(root, args)) + assert.Equal(t, "", *selector, "the selector flag value was written") + assert.False(t, root.PersistentFlags().Lookup("selector").Changed) +} diff --git a/core/om/root.go b/core/om/root.go index 3e045808b..82a0f30ff 100644 --- a/core/om/root.go +++ b/core/om/root.go @@ -159,7 +159,8 @@ func Execute() { sessioncache.PurgeCache() } -func setExecuteArgs(args []string) { +// setExecuteArgs sets the args cobra will parse, and returns them. +func setExecuteArgs(args []string) []string { var lookupArgs, cobraArgs []string // // Note: @@ -184,7 +185,7 @@ func setExecuteArgs(args []string) { lookupArgs = args cobraArgs = []string{} } else { - return + return args } _, _, err := root.Find(lookupArgs) @@ -200,8 +201,10 @@ func setExecuteArgs(args []string) { args = append(args, lookupArgs[1:]...) root.SetArgs(args) cobra.CompDebug(fmt.Sprintf("modified args: %s\n", args), false) + return args } } + return args } // ExecuteArgs parses args and executes the cobra command. @@ -214,7 +217,10 @@ func ExecuteArgs(args []string) { } var xc int var xerr exitcoder - setExecuteArgs(args) + if err := commoncmd.ValidateArgs(root, setExecuteArgs(args)); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } if err := root.Execute(); err != nil { if errors.As(err, &xerr) { xc = xerr.ExitCode() diff --git a/core/ox/root.go b/core/ox/root.go index dae74581e..685b762c9 100644 --- a/core/ox/root.go +++ b/core/ox/root.go @@ -70,7 +70,8 @@ func Execute() { ExecuteArgs(os.Args[1:]) } -func setExecuteArgs(args []string) { +// setExecuteArgs sets the args cobra will parse, and returns them. +func setExecuteArgs(args []string) []string { var lookupArgs, cobraArgs []string // // Note: @@ -117,8 +118,10 @@ func setExecuteArgs(args []string) { } root.SetArgs(args) cobra.CompDebug(fmt.Sprintf("modified args: %s\n", args), false) + return args } } + return args } // ExecuteArgs parses args and executes the cobra command. @@ -131,7 +134,10 @@ func ExecuteArgs(args []string) { } var xc int var xerr exitcoder - setExecuteArgs(args) + if err := commoncmd.ValidateArgs(root, setExecuteArgs(args)); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } if err := root.Execute(); err != nil { if errors.As(err, &xerr) { xc = xerr.ExitCode() From e69b93dcfba829bdc2d08357de0a7e80bd5016d5 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 11:45:18 +0200 Subject: [PATCH 48/54] Fix the daemon argv naming commands that no longer exist The scheduler and the api action handlers exec om with a hard-coded argv. Four of them named no command: push_resinfo om instance resource info push sync_update om instance sync update api push resinfo om instance push resinfo api sync ingest om instance sync ingest The resource subsystem hangs off the object, not off the instance, and the sync one likewise. om answered those with the instance help text and exit 0, so the hourly resinfo refresh, the scheduled sync updates, and both api actions have been silent no-ops. On a cluster where nobody ran the working spelling by hand, the resinfo cache the object and instance resource info endpoints read was never written. sync_update now passes the entry rid: those entries are created one per sync resource, from the resource schedule keyword, so the whole object was never the target. Drop the pushstats and reboot cases: they name commands om does not have either, and no schedule entry builder emits them. An unknown action at least returns an error the scheduler logs, instead of a successful run. TestSchedulerCmdArgsResolve and TestDaemonAPIExecArgsResolve resolve every one of these argv against the om command tree. Nothing else keeps them in sync: they are strings, so neither the compiler nor the type system has anything to say about them. The api ones are read back from the source, being built inline in each handler. --- core/om/args_test.go | 150 ++++++++++++++++++ .../post_instance_action_push_resinfo.go | 2 +- .../post_instance_action_sync_ingest.go | 2 +- daemon/scheduler/jobs.go | 99 ++++++++---- 4 files changed, 219 insertions(+), 34 deletions(-) create mode 100644 core/om/args_test.go diff --git a/core/om/args_test.go b/core/om/args_test.go new file mode 100644 index 000000000..c4c49127a --- /dev/null +++ b/core/om/args_test.go @@ -0,0 +1,150 @@ +package om + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/opensvc/om3/v3/core/commoncmd" + "github.com/opensvc/om3/v3/core/naming" + "github.com/opensvc/om3/v3/core/schedule" + "github.com/opensvc/om3/v3/daemon/scheduler" +) + +// requireResolves fails when args name no runnable command of the om tree. +// +// cobra prints a help text and exits 0 in that case, so an argv the daemon +// execs can rot unnoticed: this is the only thing keeping the daemon argv and +// the command tree in sync. +func requireResolves(t *testing.T, args []string) { + t.Helper() + final := setExecuteArgs(args) + require.NoErrorf(t, commoncmd.ValidateArgs(root, final), + "om %s", strings.Join(args, " ")) + cmd, _, err := root.Find(final) + require.NoErrorf(t, err, "om %s", strings.Join(args, " ")) + assert.Truef(t, cmd.Runnable(), "om %s: %q is not runnable", + strings.Join(args, " "), cmd.CommandPath()) +} + +// The daemon scheduler execs om with an argv it builds from the entry action. +func TestSchedulerCmdArgsResolve(t *testing.T) { + objectPath, err := naming.ParsePath("test/svc/s1") + require.NoError(t, err) + + for _, test := range []struct { + actions []string + path naming.Path + }{ + {actions: scheduler.ObjectActions, path: objectPath}, + {actions: scheduler.NodeActions}, + } { + for _, action := range test.actions { + t.Run(action, func(t *testing.T) { + e := schedule.Entry{ + Path: test.path, + Config: schedule.Config{ + Action: action, + Key: "task#1.schedule", + }, + } + args, err := scheduler.CmdArgs(e) + require.NoError(t, err) + requireResolves(t, args) + }) + } + } +} + +// An action scheduler.CmdArgs does not know must be an error, not an argv the +// scheduler happily execs. +func TestSchedulerCmdArgsRejectsUnknownAction(t *testing.T) { + _, err := scheduler.CmdArgs(schedule.Entry{Config: schedule.Config{Action: "no_such_action"}}) + assert.Error(t, err) +} + +// The api handlers exec om with an argv they build inline. Read them back from +// the source: there is no other enumeration of them. +func TestDaemonAPIExecArgsResolve(t *testing.T) { + args := daemonAPIExecArgs(t) + require.NotEmpty(t, args, "no api exec argv found: has the source moved?") + for _, a := range args { + t.Run(strings.Join(a, " "), func(t *testing.T) { + requireResolves(t, a) + }) + } +} + +// daemonAPIExecArgs returns the `args := []string{...}` literals of the +// daemonapi package, with the object path placeholder substituted. Elements +// that are not string literals, and the args appended conditionally, are left +// out: they are flags, never command names. +func daemonAPIExecArgs(t *testing.T) [][]string { + t.Helper() + dir := filepath.Join("..", "..", "daemon", "daemonapi") + entries, err := os.ReadDir(dir) + require.NoError(t, err) + + fset := token.NewFileSet() + var out [][]string + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0) + require.NoError(t, err) + ast.Inspect(file, func(n ast.Node) bool { + assign, ok := n.(*ast.AssignStmt) + if !ok || len(assign.Lhs) != 1 || len(assign.Rhs) != 1 { + return true + } + if ident, ok := assign.Lhs[0].(*ast.Ident); !ok || ident.Name != "args" { + return true + } + lit, ok := assign.Rhs[0].(*ast.CompositeLit) + if !ok { + return true + } + if arr, ok := lit.Type.(*ast.ArrayType); !ok { + return true + } else if ident, ok := arr.Elt.(*ast.Ident); !ok || ident.Name != "string" { + return true + } + words := make([]string, 0, len(lit.Elts)) + for _, elt := range lit.Elts { + switch v := elt.(type) { + case *ast.BasicLit: + if v.Kind != token.STRING { + return true + } + s, err := strconv.Unquote(v.Value) + if err != nil { + return true + } + words = append(words, s) + case *ast.CallExpr: + // p.String(), the object path + words = append(words, "test/svc/s1") + default: + // something we can not evaluate: skip the whole argv + // rather than test a truncated one + return true + } + } + if len(words) > 0 { + out = append(out, words) + } + return true + }) + } + return out +} diff --git a/daemon/daemonapi/post_instance_action_push_resinfo.go b/daemon/daemonapi/post_instance_action_push_resinfo.go index a9ba7ba2f..b8261ab79 100644 --- a/daemon/daemonapi/post_instance_action_push_resinfo.go +++ b/daemon/daemonapi/post_instance_action_push_resinfo.go @@ -32,7 +32,7 @@ func (a *DaemonAPI) postLocalInstanceActionPushResourceInfo(ctx echo.Context, na return JSONProblemf(ctx, http.StatusBadRequest, "Invalid parameters", "%s", err) } log = naming.LogWithPath(log, p) - args := []string{p.String(), "instance", "push", "resinfo"} + args := []string{p.String(), "resource", "info", "push"} if params.SessionId != nil { requesterSid = *params.SessionId } diff --git a/daemon/daemonapi/post_instance_action_sync_ingest.go b/daemon/daemonapi/post_instance_action_sync_ingest.go index a9fc8ef00..bd5aafaac 100644 --- a/daemon/daemonapi/post_instance_action_sync_ingest.go +++ b/daemon/daemonapi/post_instance_action_sync_ingest.go @@ -45,7 +45,7 @@ func (a *DaemonAPI) postLocalInstanceActionSyncIngest(ctx echo.Context, namespac return JSONProblemf(ctx, http.StatusBadRequest, "Invalid parameters", "%s", err) } log = naming.LogWithPath(log, p) - args := []string{p.String(), "instance", "sync", "ingest"} + args := []string{p.String(), "sync", "ingest"} if params.Rid != nil && *params.Rid != "" { args = append(args, "--rid", *params.Rid) } diff --git a/daemon/scheduler/jobs.go b/daemon/scheduler/jobs.go index 66531a63b..4fcd258be 100644 --- a/daemon/scheduler/jobs.go +++ b/daemon/scheduler/jobs.go @@ -3,6 +3,7 @@ package scheduler import ( "fmt" "os" + "slices" "time" "github.com/opensvc/om3/v3/daemon/proc" @@ -16,49 +17,83 @@ import ( "github.com/opensvc/om3/v3/util/xsession" ) -func (o *T) action(e schedule.Entry) error { - logger := o.jobLogger(e) - eid := xsession.NewEid() - sid := xsession.NewSid() - labels := []pubsub.Label{{"node", o.localhost}, {"origin", "scheduler"}} - cmdArgs := []string{} - if e.Path.IsZero() { - cmdArgs = append(cmdArgs, "node") +// NodeActions and ObjectActions list the schedule entry actions CmdArgs knows +// how to run, split by the scope of the om command each one runs. +var ( + NodeActions = []string{ + "checks", + "compliance_auto", + "pushasset", + "pushdisks", + "pushpkg", + "sysreport", + } + ObjectActions = []string{ + "push_resinfo", + "resource_monitor", + "run", + "status", + "sync_update", + } +) + +// CmdArgs returns the om argv the scheduler runs for the entry. +// +// These words are a contract with the om command tree, and nothing but that +// tree enforces it: an argv naming no command has om print a help text and +// exit 0, which the scheduler reports as a successful run. core/om's +// TestSchedulerCmdArgsResolve keeps this function and the tree in sync. +func CmdArgs(e schedule.Entry) ([]string, error) { + var head, tail []string + + if slices.Contains(NodeActions, e.Action) { + head = []string{"node"} } else { - p := e.Path.String() - cmdArgs = append(cmdArgs, p, "instance") - labels = append(labels, pubsub.Label{"namespace", e.Path.Namespace}, pubsub.Label{"path", p}) + head = []string{e.Path.String()} } + switch e.Action { case "status": - cmdArgs = append(cmdArgs, "status", "-r") + tail = []string{"instance", "status", "-r"} case "resource_monitor": - cmdArgs = append(cmdArgs, "status", "-m") + tail = []string{"instance", "status", "-m"} case "push_resinfo": - cmdArgs = append(cmdArgs, "resource", "info", "push") + tail = []string{"resource", "info", "push"} case "run": - cmdArgs = append(cmdArgs, "run", "--rid", e.RID()) + tail = []string{"instance", "run", "--rid", e.RID()} + case "sync_update": + tail = []string{"instance", "update", "--rid", e.RID()} case "pushasset": - cmdArgs = append(cmdArgs, "push", "asset") - case "reboot": - cmdArgs = append(cmdArgs, "reboot") - case "checks": - cmdArgs = append(cmdArgs, "checks") - case "compliance_auto": - cmdArgs = append(cmdArgs, "compliance", "auto") + tail = []string{"push", "asset"} case "pushdisks": - cmdArgs = append(cmdArgs, "push", "disk") + tail = []string{"push", "disk"} case "pushpkg": - cmdArgs = append(cmdArgs, "push", "pkg") - case "pushstats": - cmdArgs = append(cmdArgs, "push", "stats") + tail = []string{"push", "pkg"} + case "checks": + tail = []string{"checks"} + case "compliance_auto": + tail = []string{"compliance", "auto"} case "sysreport": - cmdArgs = append(cmdArgs, "sysreport") - case "sync_update": - cmdArgs = append(cmdArgs, "sync", "update") + tail = []string{"sysreport"} default: - logger.Errorf("unknown scheduler action") - return fmt.Errorf("unknown scheduler action") + return nil, fmt.Errorf("unknown scheduler action: %s", e.Action) + } + + return append(head, tail...), nil +} + +func (o *T) action(e schedule.Entry) error { + logger := o.jobLogger(e) + eid := xsession.NewEid() + sid := xsession.NewSid() + labels := []pubsub.Label{{"node", o.localhost}, {"origin", "scheduler"}} + if !e.Path.IsZero() { + labels = append(labels, pubsub.Label{"namespace", e.Path.Namespace}, pubsub.Label{"path", e.Path.String()}) + } + cmdArgs, err := CmdArgs(e) + if err != nil { + logger.Errorf("%s", err) + return err } var cmdEnv []string cmdEnv = append( @@ -109,7 +144,7 @@ func (o *T) action(e schedule.Entry) error { Cmd: cmd.String(), Rid: e.RID(), }) - err := cmd.Wait() + err = cmd.Wait() proc.Unregister(pid) if err != nil { duration := time.Now().Sub(startTime) From 7b4df88c648dc578711d49b539daf741a75d54cf Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 11:59:51 +0200 Subject: [PATCH 49/54] Drop the empty group sections from the command help "om svc resource --help" printed an empty "Subsystems:" section, and 24 other sections across the tree were empty too: the object print commands with their hidden children only, the pg commands, and the instance commands of the kinds having no replication resource. Two causes. The usage template prints a group title for every group a command declares, whether or not a command of that command belongs to it. That is inherited from the cobra default template. Ask first. And "resource info", holding the list and push commands, declared no group at all. It landed in the "Additional Commands" section, right below the "Subsystems:" title it was meant to fill. The help of the 22 commands with populated groups lists exactly what it listed before, empty titles aside. --- core/commoncmd/cobra_helpers.go | 23 ++++++- core/commoncmd/object.go | 5 +- core/om/help_test.go | 104 ++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 core/om/help_test.go diff --git a/core/commoncmd/cobra_helpers.go b/core/commoncmd/cobra_helpers.go index be6bb4cb3..997723463 100644 --- a/core/commoncmd/cobra_helpers.go +++ b/core/commoncmd/cobra_helpers.go @@ -47,6 +47,25 @@ func formatArgText(arg string) string { return strings.Join(lines, "\n") } +func init() { + cobra.AddTemplateFunc("hasGroupCommands", hasGroupCommands) +} + +// hasGroupCommands is true when at least one command of cmd shows up in the +// group. The usage template asks before printing a group title, so a group +// holding no command, or only hidden ones, leaves no empty section behind. +func hasGroupCommands(cmd *cobra.Command, groupID string) bool { + for _, sub := range cmd.Commands() { + if sub.GroupID != groupID { + continue + } + if sub.IsAvailableCommand() || sub.Name() == "help" { + return true + } + } + return false +} + // usageTemplate is the custom template that conditionally includes Arguments section // Note: We cannot use nindent as it's a Cobra internal function, so we rely on the fact // that the data is already properly formatted @@ -64,10 +83,10 @@ Examples: {{.Example}}{{end}}{{if .HasAvailableSubCommands}}{{$cmds := .Commands}}{{if eq (len .Groups) 0}} Available Commands:{{range $cmds}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{else}}{{range $group := .Groups}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{else}}{{range $group := .Groups}}{{if hasGroupCommands $ $group.ID}} {{.Title}}{{range $cmds}}{{if (and (eq .GroupID $group.ID) (or .IsAvailableCommand (eq .Name "help")))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if not .AllChildCommandsHaveGroup}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{end}}{{if not .AllChildCommandsHaveGroup}} Additional Commands:{{range $cmds}}{{if (and (eq .GroupID "") (or .IsAvailableCommand (eq .Name "help")))}} {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{end}}{{end}}{{if .Annotations.args}} diff --git a/core/commoncmd/object.go b/core/commoncmd/object.go index fc9fbabe7..610dae2a3 100644 --- a/core/commoncmd/object.go +++ b/core/commoncmd/object.go @@ -210,8 +210,9 @@ func NewCmdObjectResource(kind string) *cobra.Command { func NewCmdObjectResourceInfo(kind string) *cobra.Command { return &cobra.Command{ - Use: "info", - Short: "list, push the key-values reported by resources", + GroupID: GroupIDSubsystems, + Use: "info", + Short: "list, push the key-values reported by resources", } } diff --git a/core/om/help_test.go b/core/om/help_test.go new file mode 100644 index 000000000..f9821644a --- /dev/null +++ b/core/om/help_test.go @@ -0,0 +1,104 @@ +package om + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + + "github.com/opensvc/om3/v3/core/commoncmd" +) + +// walkCommands calls f on cmd and on every command below it. +func walkCommands(cmd *cobra.Command, f func(*cobra.Command)) { + f(cmd) + for _, sub := range cmd.Commands() { + walkCommands(sub, f) + } +} + +// visibleGroupCommands returns the names of the commands of cmd the usage +// template prints under the group. +func visibleGroupCommands(cmd *cobra.Command, groupID string) []string { + var l []string + for _, sub := range cmd.Commands() { + if sub.GroupID != groupID { + continue + } + if sub.IsAvailableCommand() || sub.Name() == "help" { + l = append(l, sub.Name()) + } + } + return l +} + +// A command declaring a group no command of it belongs to used to print the +// group title with nothing under it: "om svc resource --help" had an empty +// "Subsystems:" section. +func TestHelpHasNoEmptyGroupSection(t *testing.T) { + walkCommands(root, func(cmd *cobra.Command) { + groups := cmd.Groups() + if len(groups) == 0 { + return + } + usage := cmd.UsageString() + for _, group := range groups { + if len(visibleGroupCommands(cmd, group.ID)) > 0 { + continue + } + assert.NotContainsf(t, usage, group.Title, + "%q prints the empty %q section", cmd.CommandPath(), group.Title) + } + }) +} + +// The group sections must not lose a command on the way either. +func TestHelpListsEveryVisibleCommand(t *testing.T) { + walkCommands(root, func(cmd *cobra.Command) { + if !cmd.HasAvailableSubCommands() { + return + } + usage := cmd.UsageString() + for _, sub := range cmd.Commands() { + if !sub.IsAvailableCommand() { + continue + } + assert.Containsf(t, usage, " "+sub.Name()+" ", + "%q does not list %q", cmd.CommandPath(), sub.Name()) + } + }) +} + +// A command holding subcommands is a subsystem, and belongs to that group +// when its parent sorts its commands in groups. Left ungrouped, it lands in +// the "Additional Commands" section, below the empty group title it should +// have filled. +func TestSubsystemCommandsAreGrouped(t *testing.T) { + for _, kind := range []string{"svc", "vol", "all"} { + cmd, _, err := root.Find([]string{kind, "resource", "info"}) + if !assert.NoErrorf(t, err, "om %s resource info", kind) { + continue + } + assert.Equalf(t, "info", cmd.Name(), "om %s resource info", kind) + assert.Equalf(t, commoncmd.GroupIDSubsystems, cmd.GroupID, + "om %s resource info is not in the subsystems group", kind) + } +} + +// TestHelpSectionsAreNotEmpty guards the section titles the template prints +// outside of the groups. +func TestHelpSectionsAreNotEmpty(t *testing.T) { + walkCommands(root, func(cmd *cobra.Command) { + usage := cmd.UsageString() + for _, section := range []string{"Additional Commands:", "Available Commands:", "Arguments:", "Flags:"} { + i := strings.Index(usage, section) + if i < 0 { + continue + } + rest := strings.TrimLeft(usage[i+len(section):], "\n") + assert.NotEmptyf(t, strings.TrimSpace(strings.SplitN(rest, "\n", 2)[0]), + "%q prints the empty %q section", cmd.CommandPath(), section) + } + }) +} From 037ee350be889c9e354b6f02c1d4d8b7275eb479 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 12:11:47 +0200 Subject: [PATCH 50/54] Sort the remaining subsystem commands in their parent help section node push, sec and usr certificate, and the daemon dns, hb, listener and relay commands hold subcommands but belonged to no section: they landed in the "Additional Commands" one, among the verbs of their parent. The daemon command had no subsystems section at all to put them in. TestSubsystemCommandsAreGrouped now walks the tree instead of naming the resource info command: wherever a command offers a subsystems section, the commands holding subcommands under it must belong to a section, be it that one or the resource groups one. The root command is left out: its object kinds and its subsystems are all still ungrouped, and naming their sections is a question of its own. Two bugs found on the way, both of them a copy-pasted constructor: - "om daemon" registered two run commands, the running one having been built from the run constructor. Cobra runs the first of the two, so "om daemon running", which exits 0 when the daemon runs and 1 when it does not, could not be reached at all. - the instance unprovision command aliased itself "prov", the alias of the provision command. "om instance unprov" named no command, and "prov" named two. TestNoDuplicateCommandName covers both. It skips the hidden commands a visible one shadows: those are backward compatibility spellings, not duplicates. --- core/commoncmd/daemon.go | 20 ++++++++------ core/om/daemon.go | 1 + core/om/factory.go | 5 ++-- core/om/help_test.go | 59 ++++++++++++++++++++++++++++++++-------- core/om/node.go | 5 ++-- core/ox/daemon.go | 1 + core/ox/factory.go | 3 +- core/ox/node.go | 5 ++-- 8 files changed, 72 insertions(+), 27 deletions(-) diff --git a/core/commoncmd/daemon.go b/core/commoncmd/daemon.go index 1736e0b87..2b1d44c0f 100644 --- a/core/commoncmd/daemon.go +++ b/core/commoncmd/daemon.go @@ -11,29 +11,33 @@ func NewCmdDaemon() *cobra.Command { func NewCmdDaemonDNS() *cobra.Command { return &cobra.Command{ - Use: "dns", - Short: "manage the nameserver", + GroupID: GroupIDSubsystems, + Use: "dns", + Short: "manage the nameserver", } } func NewCmdDaemonHeartbeat() *cobra.Command { return &cobra.Command{ - Use: "hb", - Short: "manage heartbeats", + GroupID: GroupIDSubsystems, + Use: "hb", + Short: "manage heartbeats", } } func NewCmdDaemonListener() *cobra.Command { return &cobra.Command{ - Use: "listener", - Short: "manage listeners", + GroupID: GroupIDSubsystems, + Use: "listener", + Short: "manage listeners", } } func NewCmdDaemonRelay() *cobra.Command { return &cobra.Command{ - Use: "relay", - Short: "manage the relay server", + GroupID: GroupIDSubsystems, + Use: "relay", + Short: "manage the relay server", } } diff --git a/core/om/daemon.go b/core/om/daemon.go index ce9516fdd..2da08f5c5 100644 --- a/core/om/daemon.go +++ b/core/om/daemon.go @@ -18,6 +18,7 @@ func init() { cmdDaemon.AddGroup( commoncmd.NewGroupQuery(), + commoncmd.NewGroupSubsystems(), ) cmdDaemon.AddCommand( cmdDaemonDNS, diff --git a/core/om/factory.go b/core/om/factory.go index 5ce70c350..b8db2ab78 100644 --- a/core/om/factory.go +++ b/core/om/factory.go @@ -118,7 +118,7 @@ func newCmdDaemonRun() *cobra.Command { func newCmdDaemonRunning() *cobra.Command { var options commands.CmdDaemonRunning - cmd := commoncmd.NewCmdDaemonRun() + cmd := commoncmd.NewCmdDaemonRunning() cmd.RunE = func(cmd *cobra.Command, args []string) error { return options.Run() } @@ -1383,6 +1383,7 @@ func newCmdNodeUpdateSSHKeys() *cobra.Command { func newCmdObjectCertificate(kind string) *cobra.Command { return &cobra.Command{ + GroupID: commoncmd.GroupIDSubsystems, Aliases: []string{"cert", "crt"}, Use: "certificate", Short: "create, renew, delete certificates", @@ -3047,7 +3048,7 @@ func newCmdObjectInstanceUnprovision(kind string) *cobra.Command { Use: "unprovision", Short: "free the system resources of the instance resources (data-loss danger)", Long: "Free the system resources required by the object instance resources.\n\nOperate on a selection of instances asynchronously using --node=.", - Aliases: []string{"prov"}, + Aliases: []string{"unprov"}, RunE: func(cmd *cobra.Command, args []string) error { return options.Run(kind) }, diff --git a/core/om/help_test.go b/core/om/help_test.go index f9821644a..0e0519dde 100644 --- a/core/om/help_test.go +++ b/core/om/help_test.go @@ -1,6 +1,7 @@ package om import ( + "slices" "strings" "testing" @@ -70,20 +71,54 @@ func TestHelpListsEveryVisibleCommand(t *testing.T) { }) } -// A command holding subcommands is a subsystem, and belongs to that group -// when its parent sorts its commands in groups. Left ungrouped, it lands in -// the "Additional Commands" section, below the empty group title it should -// have filled. +// A command holding subcommands belongs to a section of its parent help, be +// it the subsystems one or the resource groups one. Left ungrouped it lands in +// the "Additional Commands" section, among the verbs of the parent, and below +// the section title it should have filled. +// +// Only the parents offering a subsystems section are checked: the root command +// sorts none of its object kinds and subsystems yet. func TestSubsystemCommandsAreGrouped(t *testing.T) { - for _, kind := range []string{"svc", "vol", "all"} { - cmd, _, err := root.Find([]string{kind, "resource", "info"}) - if !assert.NoErrorf(t, err, "om %s resource info", kind) { - continue + walkCommands(root, func(cmd *cobra.Command) { + if !slices.ContainsFunc(cmd.Groups(), func(g *cobra.Group) bool { + return g.ID == commoncmd.GroupIDSubsystems + }) { + return } - assert.Equalf(t, "info", cmd.Name(), "om %s resource info", kind) - assert.Equalf(t, commoncmd.GroupIDSubsystems, cmd.GroupID, - "om %s resource info is not in the subsystems group", kind) - } + for _, sub := range cmd.Commands() { + if !sub.IsAvailableCommand() || !sub.HasAvailableSubCommands() { + continue + } + assert.NotEmptyf(t, sub.GroupID, + "%q holds commands but no section of %q lists it", + sub.CommandPath(), cmd.CommandPath()) + } + }) +} + +// Two commands sharing a name under the same parent: cobra runs the first one +// and the second is unreachable, its help line printed twice. "om daemon" had +// two "run" commands, the running one having been built from the run +// constructor. +func TestNoDuplicateCommandName(t *testing.T) { + walkCommands(root, func(cmd *cobra.Command) { + seen := make(map[string]string) + for _, sub := range cmd.Commands() { + if !sub.IsAvailableCommand() { + // a hidden command a visible one shadows is a backward + // compatibility spelling, not a duplicate + continue + } + names := append([]string{sub.Name()}, sub.Aliases...) + slices.Sort(names) + for _, name := range slices.Compact(names) { + assert.Emptyf(t, seen[name], + "%q calls both %q and %q %q", + cmd.CommandPath(), seen[name], sub.Name(), name) + seen[name] = sub.Name() + } + } + }) } // TestHelpSectionsAreNotEmpty guards the section titles the template prints diff --git a/core/om/node.go b/core/om/node.go index f7a11d024..35c37abd2 100644 --- a/core/om/node.go +++ b/core/om/node.go @@ -57,8 +57,9 @@ var ( Aliases: []string{"prin", "pri", "pr"}, } cmdNodePush = &cobra.Command{ - Use: "push", - Short: "push node discover information to the collector", + GroupID: commoncmd.GroupIDSubsystems, + Use: "push", + Short: "push node discover information to the collector", } cmdNodeUpdate = &cobra.Command{ Use: "update", diff --git a/core/ox/daemon.go b/core/ox/daemon.go index 80b59c4c0..6c0ed71d9 100644 --- a/core/ox/daemon.go +++ b/core/ox/daemon.go @@ -16,6 +16,7 @@ func init() { ) cmdDaemon.AddGroup( commoncmd.NewGroupQuery(), + commoncmd.NewGroupSubsystems(), ) cmdDaemon.AddCommand( cmdDaemonDNS, diff --git a/core/ox/factory.go b/core/ox/factory.go index 1244ab53e..7fb9a6717 100644 --- a/core/ox/factory.go +++ b/core/ox/factory.go @@ -1329,6 +1329,7 @@ func newCmdNodeSSHTrust() *cobra.Command { func newCmdObjectCertificate(kind string) *cobra.Command { return &cobra.Command{ + GroupID: commoncmd.GroupIDSubsystems, Aliases: []string{"cert", "crt"}, Use: "certificate", Short: "create, renew, delete certificates", @@ -3292,7 +3293,7 @@ func newCmdObjectInstanceUnprovision(kind string) *cobra.Command { Use: "unprovision", Short: "free the system resources of the instance resources (data-loss danger)", Long: "Free the system resources required by the object instance resources.\n\nOperate on a selection of instances asynchronously using --node=.", - Aliases: []string{"prov"}, + Aliases: []string{"unprov"}, RunE: func(cmd *cobra.Command, args []string) error { return options.Run(kind) }, diff --git a/core/ox/node.go b/core/ox/node.go index 2508f0cc0..564a83545 100644 --- a/core/ox/node.go +++ b/core/ox/node.go @@ -44,8 +44,9 @@ var ( Aliases: []string{"prin", "pri", "pr"}, } cmdNodePush = &cobra.Command{ - Use: "push", - Short: "push node discover information to the collector", + GroupID: commoncmd.GroupIDSubsystems, + Use: "push", + Short: "push node discover information to the collector", } cmdNodeComplianceAttach = &cobra.Command{ Use: "attach", From ce4130ea394dccee3bdcc8c8810bced51d590830 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 12:16:54 +0200 Subject: [PATCH 51/54] Run the instance ingest command on the api sync ingest action The sync ingest command carries a sync resource selector by default, where the ingest action is not the sole business of the sync resources: a disk resource group can support it too. The instance ingest command selects them all, and the handler passes the rid, subset and tag parameters through, so the caller keeps narrowing the selection down. --- daemon/daemonapi/post_instance_action_sync_ingest.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/daemon/daemonapi/post_instance_action_sync_ingest.go b/daemon/daemonapi/post_instance_action_sync_ingest.go index bd5aafaac..41937c62a 100644 --- a/daemon/daemonapi/post_instance_action_sync_ingest.go +++ b/daemon/daemonapi/post_instance_action_sync_ingest.go @@ -45,7 +45,9 @@ func (a *DaemonAPI) postLocalInstanceActionSyncIngest(ctx echo.Context, namespac return JSONProblemf(ctx, http.StatusBadRequest, "Invalid parameters", "%s", err) } log = naming.LogWithPath(log, p) - args := []string{p.String(), "sync", "ingest"} + // "instance ingest", not "sync ingest": the ingest action is not the sole + // business of the sync resources, and no rid means every rid. + args := []string{p.String(), "instance", "ingest"} if params.Rid != nil && *params.Rid != "" { args = append(args, "--rid", *params.Rid) } From 39b8d290bbac6158e4d3e719ccca773af556fcd8 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 14:33:37 +0200 Subject: [PATCH 52/54] Drop the duplicate ox monitor command The ccfg command file added a monitor command to the ox root, which the monitor command file already does: the ox help listed monitor twice, and cobra ran the first of the two. --- core/ox/kind_ccfg.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/ox/kind_ccfg.go b/core/ox/kind_ccfg.go index c295e6772..c3e9125b1 100644 --- a/core/ox/kind_ccfg.go +++ b/core/ox/kind_ccfg.go @@ -16,7 +16,6 @@ func init() { root.AddCommand( cmdObject, - commoncmd.NewCmdMonitor(), ) cmdObject.AddGroup( commoncmd.NewGroupOrchestrated(), From f78dbaeb5d2e5c3d0fcd82636e0ca3ba2d4d47a9 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 14:33:43 +0200 Subject: [PATCH 53/54] Sort the root commands in help sections The om and ox root help listed every command in a single flat section, where the object kinds, the subsystems and the query commands are three different things to reach for. The root command now declares the three sections. It is built from a package level variable initializer, which the runtime runs before every init() of the package, so the command files register into those sections whatever order they run in. --- core/commoncmd/daemon.go | 5 ++-- core/commoncmd/factory_kind.go | 48 ++++++++++++++++++++-------------- core/commoncmd/groups.go | 10 +++++++ core/commoncmd/node.go | 5 ++-- core/om/array.go | 7 ++--- core/om/monitor.go | 3 --- core/om/root.go | 18 +++++++++++-- core/ox/array.go | 7 ++--- core/ox/factory.go | 14 ++++++---- core/ox/monitor.go | 3 --- core/ox/root.go | 18 +++++++++++-- 11 files changed, 94 insertions(+), 44 deletions(-) diff --git a/core/commoncmd/daemon.go b/core/commoncmd/daemon.go index 2b1d44c0f..0fdd52746 100644 --- a/core/commoncmd/daemon.go +++ b/core/commoncmd/daemon.go @@ -4,8 +4,9 @@ import "github.com/spf13/cobra" func NewCmdDaemon() *cobra.Command { return &cobra.Command{ - Use: "daemon", - Short: "manage the daemon and its components", + GroupID: GroupIDSubsystems, + Use: "daemon", + Short: "manage the daemon and its components", } } diff --git a/core/commoncmd/factory_kind.go b/core/commoncmd/factory_kind.go index f5e36dbba..39de955be 100644 --- a/core/commoncmd/factory_kind.go +++ b/core/commoncmd/factory_kind.go @@ -7,16 +7,18 @@ import ( // NewCmdAll creates the "all" command func NewCmdAll() *cobra.Command { return &cobra.Command{ - Use: "all", - Short: "manage a mix of objects, tentatively exposing all commands", + GroupID: GroupIDObjectKinds, + Use: "all", + Short: "manage a mix of objects, tentatively exposing all commands", } } // NewCmdCcfg creates the "ccfg" command func NewCmdCcfg() *cobra.Command { return &cobra.Command{ - Use: "ccfg", - Short: "manage the cluster shared configuration", + GroupID: GroupIDObjectKinds, + Use: "ccfg", + Short: "manage the cluster shared configuration", Long: `The cluster nodes merge their private configuration over the cluster shared configuration. @@ -29,8 +31,9 @@ eventually replicated).`, // NewCmdCfg creates the "cfg" command func NewCmdCfg() *cobra.Command { return &cobra.Command{ - Use: "cfg", - Short: "manage configmaps", + GroupID: GroupIDObjectKinds, + Use: "cfg", + Short: "manage configmaps", Long: `A configmap is an unencrypted key-value store. Values can be binary or text. @@ -52,8 +55,9 @@ when installing the key in a volume.`, // NewCmdSec creates the "sec" command func NewCmdSec() *cobra.Command { return &cobra.Command{ - Use: "sec", - Short: "manage secrets", + GroupID: GroupIDObjectKinds, + Use: "sec", + Short: "manage secrets", Long: `A secret is an encrypted key-value store. Values can be binary or text. @@ -75,8 +79,9 @@ when installing the key in a volume.`, // NewCmdSVC creates the "svc" command func NewCmdSVC() *cobra.Command { return &cobra.Command{ - Use: "svc", - Short: "manage services", + GroupID: GroupIDObjectKinds, + Use: "svc", + Short: "manage services", Long: `Service objects subsystem. A service is typically made of ip, app, container and task resources. @@ -89,8 +94,9 @@ isolate lifecycles or to abstract cluster-specific knowledge.`, // NewCmdUsr creates the "usr" command func NewCmdUsr() *cobra.Command { return &cobra.Command{ - Use: "usr", - Short: "manage users", + GroupID: GroupIDObjectKinds, + Use: "usr", + Short: "manage users", Long: `A user stores the grants and credentials of user of the agent API. User objects are not necessary with OpenID authentication, as the @@ -101,8 +107,9 @@ grants are embedded in the trusted bearer tokens.`, // NewCmdVol creates the "vol" command func NewCmdVol() *cobra.Command { return &cobra.Command{ - Use: "vol", - Short: "manage volumes", + GroupID: GroupIDObjectKinds, + Use: "vol", + Short: "manage volumes", Long: `A volume is a persistent data provider. A volume is made of disk, fs and sync resources. It is created by a pool, @@ -115,14 +122,16 @@ Volumes and their subdirectories can be mounted inside containers.`, // NewCmdNscfg creates the "nscfg" command func NewCmdNscfg() *cobra.Command { return &cobra.Command{ - Use: "nscfg", - Short: "manage namespace configurations", + GroupID: GroupIDObjectKinds, + Use: "nscfg", + Short: "manage namespace configurations", } } // NewCmdNetwork creates the "network" command func NewCmdNetwork() *cobra.Command { return &cobra.Command{ + GroupID: GroupIDSubsystems, Use: "network", Short: "manage backend networks", Aliases: []string{"net"}, @@ -141,9 +150,10 @@ func NewCmdNetworkIP() *cobra.Command { // NewCmdPool creates the "pool" command func NewCmdPool() *cobra.Command { return &cobra.Command{ - Use: "pool", - Short: "manage storage pools", - Long: " A pool is a vol provider. Pools abstract the hardware and software specificities of the cluster infrastructure.", + GroupID: GroupIDSubsystems, + Use: "pool", + Short: "manage storage pools", + Long: " A pool is a vol provider. Pools abstract the hardware and software specificities of the cluster infrastructure.", } } diff --git a/core/commoncmd/groups.go b/core/commoncmd/groups.go index 458913b35..56e6fd387 100644 --- a/core/commoncmd/groups.go +++ b/core/commoncmd/groups.go @@ -3,6 +3,7 @@ package commoncmd import "github.com/spf13/cobra" var ( + GroupIDObjectKinds = "object kinds" GroupIDOrchestrated = "orchestrated" GroupIDQuery = "query" GroupIDSubsystems = "subsystems" @@ -10,6 +11,15 @@ var ( GroupIDReplication = "replication" ) +// NewGroupObjectKinds returns the section listing the commands scoped to an +// object kind, at the root of the command tree. +func NewGroupObjectKinds() *cobra.Group { + return &cobra.Group{ + ID: GroupIDObjectKinds, + Title: "Object Kinds:", + } +} + func NewGroupOrchestrated() *cobra.Group { return &cobra.Group{ ID: GroupIDOrchestrated, diff --git a/core/commoncmd/node.go b/core/commoncmd/node.go index 423200269..9c8609414 100644 --- a/core/commoncmd/node.go +++ b/core/commoncmd/node.go @@ -4,8 +4,9 @@ import "github.com/spf13/cobra" func NewCmdNode() *cobra.Command { cmd := &cobra.Command{ - Use: "node", - Short: "manage a opensvc cluster node", + GroupID: GroupIDSubsystems, + Use: "node", + Short: "manage a opensvc cluster node", } cmd.AddGroup( NewGroupOrchestrated(), diff --git a/core/om/array.go b/core/om/array.go index bb890e941..9d52719b5 100644 --- a/core/om/array.go +++ b/core/om/array.go @@ -15,9 +15,10 @@ import ( var ( arrayName string cmdArray = &cobra.Command{ - Use: "array", - Short: "manage storage arrays", - Long: `A array is a backend storage provider for pools.`, + GroupID: commoncmd.GroupIDSubsystems, + Use: "array", + Short: "manage storage arrays", + Long: `A array is a backend storage provider for pools.`, RunE: func(_ *cobra.Command, args []string) error { return runArray(args) }, diff --git a/core/om/monitor.go b/core/om/monitor.go index 60eac8ad1..db94d785b 100644 --- a/core/om/monitor.go +++ b/core/om/monitor.go @@ -3,9 +3,6 @@ package om import "github.com/opensvc/om3/v3/core/commoncmd" func init() { - root.AddGroup( - commoncmd.NewGroupQuery(), - ) root.AddCommand( commoncmd.NewCmdMonitor(), ) diff --git a/core/om/root.go b/core/om/root.go index 82a0f30ff..67aa25360 100644 --- a/core/om/root.go +++ b/core/om/root.go @@ -41,7 +41,15 @@ var ( //go:embed bash_completion.sh bashCompletionFunction string - root = &cobra.Command{ + root = newRootCmd() +) + +// newRootCmd builds the root command and declares the sections its commands +// sort into. It is called from a package level variable initializer, which the +// runtime runs before every init() of the package: the command files can +// register into those sections whatever order they run in. +func newRootCmd() *cobra.Command { + cmd := &cobra.Command{ Use: filepath.Base(os.Args[0]), Short: "the opensvc cluster management command", PersistentPreRunE: persistentPreRunE, @@ -51,7 +59,13 @@ var ( BashCompletionFunction: bashCompletionFunction, Version: version.Version(), } -) + cmd.AddGroup( + commoncmd.NewGroupQuery(), + commoncmd.NewGroupObjectKinds(), + commoncmd.NewGroupSubsystems(), + ) + return cmd +} func validArgs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { //return listObjectPaths(), cobra.ShellCompDirectiveNoFileComp diff --git a/core/ox/array.go b/core/ox/array.go index 8d0c7618b..991fc939e 100644 --- a/core/ox/array.go +++ b/core/ox/array.go @@ -15,9 +15,10 @@ import ( var ( arrayName string cmdArray = &cobra.Command{ - Use: "array", - Short: "manage storage arrays", - Long: ` A array is backend storage provider for pools.`, + GroupID: commoncmd.GroupIDSubsystems, + Use: "array", + Short: "manage storage arrays", + Long: ` A array is backend storage provider for pools.`, RunE: func(_ *cobra.Command, args []string) error { return runArray(args) }, diff --git a/core/ox/factory.go b/core/ox/factory.go index 7fb9a6717..652bae9f4 100644 --- a/core/ox/factory.go +++ b/core/ox/factory.go @@ -801,8 +801,9 @@ func newCmdNodeScheduleList() *cobra.Command { func newCmdNodeSystem() *cobra.Command { cmd := &cobra.Command{ - Use: "system", - Short: "node system commands", + GroupID: commoncmd.GroupIDSubsystems, + Use: "system", + Short: "node system commands", } return cmd } @@ -1148,6 +1149,7 @@ func newCmdNodeSCSIPRKey() *cobra.Command { func newCmdObjectSchedule(kind string) *cobra.Command { cmd := &cobra.Command{ + GroupID: commoncmd.GroupIDSubsystems, Use: "schedule", Short: "object scheduler commands", Aliases: []string{"sched"}, @@ -3890,8 +3892,9 @@ func newCmdPoolVolumeList() *cobra.Command { func newCmdTUI(kind string) *cobra.Command { var options tui.Options cmd := &cobra.Command{ - Use: "tui", - Short: "interactive terminal user interface", + GroupID: commoncmd.GroupIDQuery, + Use: "tui", + Short: "interactive terminal user interface", RunE: func(cmd *cobra.Command, args []string) error { options.Selector = mergeSelector("", kind, "") return tui.Run(&options) @@ -4283,7 +4286,8 @@ func newCmdObjectGen(kind string) *cobra.Command { func NewCmdContext() *cobra.Command { return &cobra.Command{ - Use: "context", + GroupID: commoncmd.GroupIDSubsystems, + Use: "context", Long: `A context groups the namespace, authentication and endpoint metadata needed to connect and manage a remote cluster. Once configured, you can login and logout from this context.`, diff --git a/core/ox/monitor.go b/core/ox/monitor.go index f18175a50..1f813f169 100644 --- a/core/ox/monitor.go +++ b/core/ox/monitor.go @@ -3,9 +3,6 @@ package ox import "github.com/opensvc/om3/v3/core/commoncmd" func init() { - root.AddGroup( - commoncmd.NewGroupQuery(), - ) root.AddCommand( commoncmd.NewCmdMonitor(), ) diff --git a/core/ox/root.go b/core/ox/root.go index 685b762c9..1ab319cfa 100644 --- a/core/ox/root.go +++ b/core/ox/root.go @@ -27,7 +27,15 @@ var ( //go:embed bash_completion.sh bashCompletionFunction string - root = &cobra.Command{ + root = newRootCmd() +) + +// newRootCmd builds the root command and declares the sections its commands +// sort into. It is called from a package level variable initializer, which the +// runtime runs before every init() of the package: the command files can +// register into those sections whatever order they run in. +func newRootCmd() *cobra.Command { + cmd := &cobra.Command{ Use: filepath.Base(os.Args[0]), Short: "the opensvc cluster management command", SilenceUsage: true, @@ -36,7 +44,13 @@ var ( BashCompletionFunction: bashCompletionFunction, Version: version.Version(), } -) + cmd.AddGroup( + commoncmd.NewGroupQuery(), + commoncmd.NewGroupObjectKinds(), + commoncmd.NewGroupSubsystems(), + ) + return cmd +} func validArgs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return listObjectPaths(), cobra.ShellCompDirectiveNoFileComp From b74f3d7e37f21074154d295a716c91fda8e2ac6c Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 28 Aug 2026 14:33:49 +0200 Subject: [PATCH 54/54] Share the command tree help checks between om and ox The checks lived in the om test file, where the ox tree, which has its own commands, went unchecked: it had an ungrouped schedule and node system command, and a duplicate monitor. They move to a helptest package, outside of commoncmd so both root commands, which commoncmd knows nothing about, can run them on their own tree. --- core/commoncmd/help.go | 59 +++++++++++ core/commoncmd/helptest/helptest.go | 145 ++++++++++++++++++++++++++++ core/om/help_test.go | 134 +------------------------ core/ox/help_test.go | 11 +++ 4 files changed, 218 insertions(+), 131 deletions(-) create mode 100644 core/commoncmd/help.go create mode 100644 core/commoncmd/helptest/helptest.go create mode 100644 core/ox/help_test.go diff --git a/core/commoncmd/help.go b/core/commoncmd/help.go new file mode 100644 index 000000000..9acd1a01f --- /dev/null +++ b/core/commoncmd/help.go @@ -0,0 +1,59 @@ +package commoncmd + +import ( + "slices" + "strings" + + "github.com/spf13/cobra" +) + +// WalkCommands calls f on cmd and on every command below it. +func WalkCommands(cmd *cobra.Command, f func(*cobra.Command)) { + f(cmd) + for _, sub := range cmd.Commands() { + WalkCommands(sub, f) + } +} + +// GroupCommands returns the commands of cmd the usage template prints under +// the group. +func GroupCommands(cmd *cobra.Command, groupID string) []*cobra.Command { + var l []*cobra.Command + for _, sub := range cmd.Commands() { + if sub.GroupID != groupID { + continue + } + if sub.IsAvailableCommand() || sub.Name() == "help" { + l = append(l, sub) + } + } + return l +} + +// CommandNames returns the name of cmd and its aliases, deduplicated: a +// command naming itself in its own alias list is one name, not two. +func CommandNames(cmd *cobra.Command) []string { + names := append([]string{cmd.Name()}, cmd.Aliases...) + slices.Sort(names) + return slices.Compact(names) +} + +// HasGroup is true when cmd declares the section. +func HasGroup(cmd *cobra.Command, groupID string) bool { + return slices.ContainsFunc(cmd.Groups(), func(g *cobra.Group) bool { + return g.ID == groupID + }) +} + +// SectionBody returns what the usage of cmd prints under the section title, +// up to the next empty line, and whether the section is there at all. +func SectionBody(cmd *cobra.Command, title string) (string, bool) { + usage := cmd.UsageString() + i := strings.Index(usage, title) + if i < 0 { + return "", false + } + rest := strings.TrimLeft(usage[i+len(title):], "\n") + body, _, _ := strings.Cut(rest, "\n\n") + return body, true +} diff --git a/core/commoncmd/helptest/helptest.go b/core/commoncmd/helptest/helptest.go new file mode 100644 index 000000000..bec6e851e --- /dev/null +++ b/core/commoncmd/helptest/helptest.go @@ -0,0 +1,145 @@ +// Package helptest holds the checks the om and ox command trees must both +// pass. It lives outside of commoncmd so the two root commands, which +// commoncmd knows nothing about, can each run them against their own tree. +package helptest + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + + "github.com/opensvc/om3/v3/core/commoncmd" +) + +// Run runs every command tree check on root. +func Run(t *testing.T, root *cobra.Command) { + t.Helper() + t.Run("no empty group section", func(t *testing.T) { NoEmptyGroupSection(t, root) }) + t.Run("no empty section", func(t *testing.T) { NoEmptySection(t, root) }) + t.Run("every visible command listed", func(t *testing.T) { EveryVisibleCommandListed(t, root) }) + t.Run("subsystem commands grouped", func(t *testing.T) { SubsystemCommandsGrouped(t, root) }) + t.Run("no duplicate command name", func(t *testing.T) { NoDuplicateCommandName(t, root) }) + t.Run("groups declared by the parent", func(t *testing.T) { GroupsDeclaredByParent(t, root) }) +} + +// NoEmptyGroupSection checks that no command prints a section title with +// nothing under it: "om svc resource --help" used to print an empty +// "Subsystems:" section. +func NoEmptyGroupSection(t *testing.T, root *cobra.Command) { + t.Helper() + commoncmd.WalkCommands(root, func(cmd *cobra.Command) { + if len(cmd.Groups()) == 0 { + return + } + usage := cmd.UsageString() + for _, group := range cmd.Groups() { + if len(commoncmd.GroupCommands(cmd, group.ID)) > 0 { + continue + } + assert.NotContainsf(t, usage, group.Title, + "%q prints the empty %q section", cmd.CommandPath(), group.Title) + } + }) +} + +// NoEmptySection checks the section titles the usage template prints outside +// of the groups. +func NoEmptySection(t *testing.T, root *cobra.Command) { + t.Helper() + sections := []string{"Additional Commands:", "Available Commands:", "Arguments:", "Flags:"} + commoncmd.WalkCommands(root, func(cmd *cobra.Command) { + for _, section := range sections { + body, ok := commoncmd.SectionBody(cmd, section) + if !ok { + continue + } + assert.NotEmptyf(t, body, + "%q prints the empty %q section", cmd.CommandPath(), section) + } + }) +} + +// EveryVisibleCommandListed checks the sections do not lose a command on the +// way. +func EveryVisibleCommandListed(t *testing.T, root *cobra.Command) { + t.Helper() + commoncmd.WalkCommands(root, func(cmd *cobra.Command) { + if !cmd.HasAvailableSubCommands() { + return + } + usage := cmd.UsageString() + for _, sub := range cmd.Commands() { + if !sub.IsAvailableCommand() { + continue + } + assert.Containsf(t, usage, " "+sub.Name()+" ", + "%q does not list %q", cmd.CommandPath(), sub.Name()) + } + }) +} + +// SubsystemCommandsGrouped checks that a command holding subcommands belongs +// to a section of its parent usage, be it the subsystems one or the resource +// groups one. Left ungrouped it lands in the "Additional Commands" section, +// among the verbs of the parent, and below the section title it should have +// filled. +// +// Only the parents offering a subsystems section are checked. +func SubsystemCommandsGrouped(t *testing.T, root *cobra.Command) { + t.Helper() + commoncmd.WalkCommands(root, func(cmd *cobra.Command) { + if !commoncmd.HasGroup(cmd, commoncmd.GroupIDSubsystems) { + return + } + for _, sub := range cmd.Commands() { + if !sub.IsAvailableCommand() || !sub.HasAvailableSubCommands() { + continue + } + assert.NotEmptyf(t, sub.GroupID, + "%q holds commands but no section of %q lists it", + sub.CommandPath(), cmd.CommandPath()) + } + }) +} + +// NoDuplicateCommandName checks that two commands of a same parent do not +// share a name or an alias: cobra runs the first one and the second is +// unreachable, its help line printed twice. +// +// The hidden commands a visible one shadows are skipped: those are backward +// compatibility spellings, not duplicates. +func NoDuplicateCommandName(t *testing.T, root *cobra.Command) { + t.Helper() + commoncmd.WalkCommands(root, func(cmd *cobra.Command) { + seen := make(map[string]string) + for _, sub := range cmd.Commands() { + if !sub.IsAvailableCommand() { + continue + } + for _, name := range commoncmd.CommandNames(sub) { + assert.Emptyf(t, seen[name], + "%q calls both %q and %q %q", + cmd.CommandPath(), seen[name], sub.Name(), name) + seen[name] = sub.Name() + } + } + }) +} + +// GroupsDeclaredByParent checks that every command belongs to a section its +// parent declares. Cobra panics on the first Execute otherwise, wherever in +// the tree the offending command sits. +func GroupsDeclaredByParent(t *testing.T, root *cobra.Command) { + t.Helper() + commoncmd.WalkCommands(root, func(cmd *cobra.Command) { + for _, sub := range cmd.Commands() { + if sub.GroupID == "" { + continue + } + assert.Truef(t, commoncmd.HasGroup(cmd, sub.GroupID), + "%q belongs to the %q section, which %q does not declare", + sub.CommandPath(), sub.GroupID, cmd.CommandPath()) + } + }) +} diff --git a/core/om/help_test.go b/core/om/help_test.go index 0e0519dde..5d0f0233b 100644 --- a/core/om/help_test.go +++ b/core/om/help_test.go @@ -1,139 +1,11 @@ package om import ( - "slices" - "strings" "testing" - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - - "github.com/opensvc/om3/v3/core/commoncmd" + "github.com/opensvc/om3/v3/core/commoncmd/helptest" ) -// walkCommands calls f on cmd and on every command below it. -func walkCommands(cmd *cobra.Command, f func(*cobra.Command)) { - f(cmd) - for _, sub := range cmd.Commands() { - walkCommands(sub, f) - } -} - -// visibleGroupCommands returns the names of the commands of cmd the usage -// template prints under the group. -func visibleGroupCommands(cmd *cobra.Command, groupID string) []string { - var l []string - for _, sub := range cmd.Commands() { - if sub.GroupID != groupID { - continue - } - if sub.IsAvailableCommand() || sub.Name() == "help" { - l = append(l, sub.Name()) - } - } - return l -} - -// A command declaring a group no command of it belongs to used to print the -// group title with nothing under it: "om svc resource --help" had an empty -// "Subsystems:" section. -func TestHelpHasNoEmptyGroupSection(t *testing.T) { - walkCommands(root, func(cmd *cobra.Command) { - groups := cmd.Groups() - if len(groups) == 0 { - return - } - usage := cmd.UsageString() - for _, group := range groups { - if len(visibleGroupCommands(cmd, group.ID)) > 0 { - continue - } - assert.NotContainsf(t, usage, group.Title, - "%q prints the empty %q section", cmd.CommandPath(), group.Title) - } - }) -} - -// The group sections must not lose a command on the way either. -func TestHelpListsEveryVisibleCommand(t *testing.T) { - walkCommands(root, func(cmd *cobra.Command) { - if !cmd.HasAvailableSubCommands() { - return - } - usage := cmd.UsageString() - for _, sub := range cmd.Commands() { - if !sub.IsAvailableCommand() { - continue - } - assert.Containsf(t, usage, " "+sub.Name()+" ", - "%q does not list %q", cmd.CommandPath(), sub.Name()) - } - }) -} - -// A command holding subcommands belongs to a section of its parent help, be -// it the subsystems one or the resource groups one. Left ungrouped it lands in -// the "Additional Commands" section, among the verbs of the parent, and below -// the section title it should have filled. -// -// Only the parents offering a subsystems section are checked: the root command -// sorts none of its object kinds and subsystems yet. -func TestSubsystemCommandsAreGrouped(t *testing.T) { - walkCommands(root, func(cmd *cobra.Command) { - if !slices.ContainsFunc(cmd.Groups(), func(g *cobra.Group) bool { - return g.ID == commoncmd.GroupIDSubsystems - }) { - return - } - for _, sub := range cmd.Commands() { - if !sub.IsAvailableCommand() || !sub.HasAvailableSubCommands() { - continue - } - assert.NotEmptyf(t, sub.GroupID, - "%q holds commands but no section of %q lists it", - sub.CommandPath(), cmd.CommandPath()) - } - }) -} - -// Two commands sharing a name under the same parent: cobra runs the first one -// and the second is unreachable, its help line printed twice. "om daemon" had -// two "run" commands, the running one having been built from the run -// constructor. -func TestNoDuplicateCommandName(t *testing.T) { - walkCommands(root, func(cmd *cobra.Command) { - seen := make(map[string]string) - for _, sub := range cmd.Commands() { - if !sub.IsAvailableCommand() { - // a hidden command a visible one shadows is a backward - // compatibility spelling, not a duplicate - continue - } - names := append([]string{sub.Name()}, sub.Aliases...) - slices.Sort(names) - for _, name := range slices.Compact(names) { - assert.Emptyf(t, seen[name], - "%q calls both %q and %q %q", - cmd.CommandPath(), seen[name], sub.Name(), name) - seen[name] = sub.Name() - } - } - }) -} - -// TestHelpSectionsAreNotEmpty guards the section titles the template prints -// outside of the groups. -func TestHelpSectionsAreNotEmpty(t *testing.T) { - walkCommands(root, func(cmd *cobra.Command) { - usage := cmd.UsageString() - for _, section := range []string{"Additional Commands:", "Available Commands:", "Arguments:", "Flags:"} { - i := strings.Index(usage, section) - if i < 0 { - continue - } - rest := strings.TrimLeft(usage[i+len(section):], "\n") - assert.NotEmptyf(t, strings.TrimSpace(strings.SplitN(rest, "\n", 2)[0]), - "%q prints the empty %q section", cmd.CommandPath(), section) - } - }) +func TestHelp(t *testing.T) { + helptest.Run(t, root) } diff --git a/core/ox/help_test.go b/core/ox/help_test.go new file mode 100644 index 000000000..89e21d209 --- /dev/null +++ b/core/ox/help_test.go @@ -0,0 +1,11 @@ +package ox + +import ( + "testing" + + "github.com/opensvc/om3/v3/core/commoncmd/helptest" +) + +func TestHelp(t *testing.T) { + helptest.Run(t, root) +}