diff --git a/rocketpool-cli/service/config/settings-alerting.go b/rocketpool-cli/service/config/settings-alerting.go index 830e5c967..b15c5f7b3 100644 --- a/rocketpool-cli/service/config/settings-alerting.go +++ b/rocketpool-cli/service/config/settings-alerting.go @@ -25,6 +25,7 @@ var alertingParametersNativeMode = map[string]interface{}{ "alertEnabled_BeaconClientSyncComplete": nil, "alertEnabled_PortConnectivityCheck": nil, "alertEnabled_LowETHBalance": nil, + "alertEnabled_ObserveModeActive": nil, "lowETHBalanceThreshold": nil, } @@ -56,6 +57,7 @@ var alertingParametersDockerMode = map[string]interface{}{ "alertEnabled_BeaconClientSyncComplete": nil, "alertEnabled_PortConnectivityCheck": nil, "alertEnabled_LowETHBalance": nil, + "alertEnabled_ObserveModeActive": nil, "lowETHBalanceThreshold": nil, } diff --git a/rocketpool-cli/wallet/end-masquerade.go b/rocketpool-cli/wallet/end-masquerade.go index 71fc4bcb6..6038ae223 100644 --- a/rocketpool-cli/wallet/end-masquerade.go +++ b/rocketpool-cli/wallet/end-masquerade.go @@ -51,6 +51,10 @@ func endMasquerade(yes bool) error { fmt.Println("Successfully ended masquerade mode.") + if status.IsObserve { + return promptAndRestartObserveDaemons(rp, yes) + } + // Return return nil } diff --git a/rocketpool-cli/wallet/masquerade.go b/rocketpool-cli/wallet/masquerade.go index fc6cd507c..e74273227 100644 --- a/rocketpool-cli/wallet/masquerade.go +++ b/rocketpool-cli/wallet/masquerade.go @@ -53,8 +53,46 @@ func masquerade(addressFlag string, yes bool, observe bool) error { fmt.Printf("Your node is now masquerading as address %s.\n", color.LightBlue(address.Hex())) if observe { - fmt.Println("Restart the node and watchtower daemons to observe as the masquerade address.") + return promptAndRestartObserveDaemons(rp, yes) } return nil } + +// promptAndRestartObserveDaemons asks the user whether to restart the node and watchtower +// containers now, so they immediately pick up the new observe-mode state. +func promptAndRestartObserveDaemons(rp *rocketpool.Client, yes bool) error { + cfg, _, err := rp.LoadConfig() + if err != nil { + return fmt.Errorf("error loading configuration: %w", err) + } + + if cfg.IsNativeMode { + fmt.Println("Restart the node and watchtower daemons for this to take effect.") + return nil + } + + if !yes && !prompt.Confirm("Would you like to restart the node and watchtower containers now?") { + fmt.Println("Remember to restart the node and watchtower containers for this to take effect.") + return nil + } + + projectName := cfg.Smartnode.ProjectName.Value.(string) + for _, name := range []string{"node", "watchtower"} { + container := fmt.Sprintf("%s_%s", projectName, name) + fmt.Printf("Restarting %s... ", container) + response, err := rp.RestartContainer(container) + if err != nil { + fmt.Println() + return fmt.Errorf("error restarting %s: %w", container, err) + } + if response != container { + fmt.Println() + return fmt.Errorf("unexpected output while restarting %s: %s", container, response) + } + fmt.Println("done!") + } + + fmt.Println("Done!") + return nil +} diff --git a/rocketpool/node/node.go b/rocketpool/node/node.go index 2fa154d21..7b3c79001 100644 --- a/rocketpool/node/node.go +++ b/rocketpool/node/node.go @@ -47,6 +47,7 @@ const ( DistributeMinipoolsColor = color.FgHiGreen ErrorColor = color.FgRed WarningColor = color.FgYellow + ObserveWarningColor = color.FgHiRed UpdateColor = color.FgHiWhite PrestakeMegapoolValidatorColor = color.FgHiGreen StakeMegapoolValidatorColor = color.FgHiBlue @@ -176,16 +177,10 @@ func run(c *cli.Command) error { return fmt.Errorf("error getting node account: %w", err) } - if isObserveMode { - red := color.New(color.FgHiRed).SprintFunc() - fmt.Println(red("Node daemon is observing address " + nodeAccount.Address.Hex() + ".")) - fmt.Println(red("Transactions will not be submitted. Fee recipient management targets your real node address.")) - fmt.Println(red("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.")) - } - // Initialize loggers errorLog := log.NewColorLogger(ErrorColor) updateLog := log.NewColorLogger(UpdateColor) + observeLog := log.NewColorLogger(ObserveWarningColor) // Create the state provider. In live mode this is a NetworkStateManager // backed by the real EC/BC; in --network-state mode it is a @@ -348,6 +343,16 @@ func run(c *cli.Command) error { } stateLocker.UpdateState(state) + // Keep the observe mode warning visible in the logs and the alert active for as long as observe mode is in effect + if isObserveMode { + observeLog.Println("Node daemon is observing address " + nodeAccount.Address.Hex() + ".") + observeLog.Println("Transactions will not be submitted. Fee recipient management targets your real node address.") + observeLog.Println("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.") + if err := alerting.AlertObserveModeActive(cfg, nodeAccount.Address); err != nil { + errorLog.Println(err) + } + } + // Manage the fee recipient for the node if err := manageFeeRecipient.run(state); err != nil { errorLog.Println(err) diff --git a/rocketpool/watchtower/watchtower.go b/rocketpool/watchtower/watchtower.go index 6d839c067..edd6e990d 100644 --- a/rocketpool/watchtower/watchtower.go +++ b/rocketpool/watchtower/watchtower.go @@ -21,6 +21,7 @@ import ( "github.com/rocket-pool/smartnode/bindings/utils" "github.com/rocket-pool/smartnode/rocketpool/watchtower/collectors" "github.com/rocket-pool/smartnode/shared/services" + "github.com/rocket-pool/smartnode/shared/services/alerting" "github.com/rocket-pool/smartnode/shared/services/beacon" "github.com/rocket-pool/smartnode/shared/services/state" "github.com/rocket-pool/smartnode/shared/services/wallet" @@ -48,6 +49,7 @@ const ( MetricsColor = color.FgHiYellow SubmitRewardsTreeColor = color.FgHiCyan WarningColor = color.FgYellow + ObserveWarningColor = color.FgHiRed ProcessPenaltiesColor = color.FgHiMagenta CancelBondsColor = color.FgGreen CheckSoloMigrationsColor = color.FgCyan @@ -158,6 +160,7 @@ func run(c *cli.Command) error { // Initialize error logger errorLog := log.NewColorLogger(ErrorColor) updateLog := log.NewColorLogger(UpdateColor) + observeLog := log.NewColorLogger(ObserveWarningColor) // Create the state manager m := state.NewNetworkStateManager(rp, cfg.Smartnode.GetStateManagerContracts(), bc, &updateLog) @@ -168,13 +171,6 @@ func run(c *cli.Command) error { return fmt.Errorf("error getting node account: %w", err) } - if isObserveMode { - red := color.New(color.FgHiRed).SprintFunc() - fmt.Println(red("Watchtower daemon is observing address " + nodeAccount.Address.Hex() + ".")) - fmt.Println(red("Transactions will not be submitted.")) - fmt.Println(red("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.")) - } - // Initialize tasks respondChallenges, err := newRespondChallenges(c, log.NewColorLogger(RespondChallengesColor), m) if err != nil { @@ -286,6 +282,16 @@ func run(c *cli.Command) error { continue } + // Keep the observe mode warning visible in the logs and the alert active for as long as observe mode is in effect + if isObserveMode { + observeLog.Println("Watchtower daemon is observing address " + nodeAccount.Address.Hex() + ".") + observeLog.Println("Transactions will not be submitted.") + observeLog.Println("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.") + if err := alerting.AlertObserveModeActive(cfg, nodeAccount.Address); err != nil { + errorLog.Println(err) + } + } + // Run the manual rewards tree generation if err := generateRewardsTree.run(); err != nil { errorLog.Println(err) diff --git a/shared/services/alerting/alerting.go b/shared/services/alerting/alerting.go index 6855541a5..2dcb840b7 100644 --- a/shared/services/alerting/alerting.go +++ b/shared/services/alerting/alerting.go @@ -256,6 +256,31 @@ const ( ClientKindBeacon ClientKind = "Beacon" ) +// Sends an alert while the node/watchtower daemon is running in observe (masquerade) mode. +// Called on every task loop iteration for as long as observe mode is active. +// If alerting/metrics are disabled, this function does nothing. +func AlertObserveModeActive(cfg *config.RocketPoolConfig, observedAddress common.Address) error { + if !isAlertingEnabled(cfg) { + logMessage("alerting is disabled, not sending AlertObserveModeActive.") + return nil + } + + if cfg.Alertmanager.AlertEnabled_ObserveModeActive.Value != true { + logMessage("alert for ObserveModeActive is disabled, not sending.") + return nil + } + + alert := createAlert( + fmt.Sprintf("ObserveModeActive-%s", observedAddress.Hex()), + "Node is running in observe mode", + fmt.Sprintf("The node/watchtower daemon is observing address %s and will not submit transactions. Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.", observedAddress.Hex()), + SeverityWarning, + strfmt.DateTime(time.Now().Add(DefaultEndsAtDurationForSeverityInfo)), + map[string]string{"address": observedAddress.Hex()}, + ) + return sendAlert(alert, cfg) +} + func alertClientSyncComplete(cfg *config.RocketPoolConfig, client ClientKind) error { alertName := fmt.Sprintf("%sClientSyncComplete", client) if !isAlertingEnabled(cfg) { diff --git a/shared/services/config/alertmanager-config.go b/shared/services/config/alertmanager-config.go index 17b9e4eb1..1a5172e8e 100644 --- a/shared/services/config/alertmanager-config.go +++ b/shared/services/config/alertmanager-config.go @@ -85,6 +85,8 @@ type AlertmanagerConfig struct { AlertEnabled_BeaconClientSyncComplete config.Parameter `yaml:"alertEnabled_BeaconClientSyncComplete,omitempty"` // Whether to periodically check if the eth1 and eth2 P2P ports are open to the internet AlertEnabled_PortConnectivityCheck config.Parameter `yaml:"alertEnabled_PortConnectivityCheck,omitempty"` + // Whether to alert while the node/watchtower daemon is running in observe (masquerade) mode + AlertEnabled_ObserveModeActive config.Parameter `yaml:"alertEnabled_ObserveModeActive,omitempty"` } func NewAlertmanagerConfig(cfg *RocketPoolConfig) *AlertmanagerConfig { @@ -286,6 +288,10 @@ func NewAlertmanagerConfig(cfg *RocketPoolConfig) *AlertmanagerConfig { "LowETHBalance", "Low ETH Balance"), + AlertEnabled_ObserveModeActive: createParameterForAlertEnablement( + "ObserveModeActive", + "the node/watchtower daemon is running in observe mode"), + LowETHBalanceThreshold: config.Parameter{ ID: "lowETHBalanceThreshold", Name: "Low ETH Balance Threshold", @@ -344,6 +350,7 @@ func (cfg *AlertmanagerConfig) GetParameters() []*config.Parameter { &cfg.AlertEnabled_BeaconClientSyncComplete, &cfg.AlertEnabled_PortConnectivityCheck, &cfg.AlertEnabled_LowETHBalance, + &cfg.AlertEnabled_ObserveModeActive, &cfg.LowETHBalanceThreshold, } }