Add alertmanager condition for observe mode.

rocketpool-cli/service/config/settings-alerting.go

@@ -25,6 +25,7 @@ var alertingParametersNativeMode = map[string]interface{}{
 	"alertEnabled_BeaconClientSyncComplete":    nil,
 	"alertEnabled_PortConnectivityCheck":       nil,
 	"alertEnabled_LowETHBalance":               nil,
+	"alertEnabled_ObserveModeActive":           nil,
 	"lowETHBalanceThreshold":                   nil,
 }
 
@@ -56,6 +57,7 @@ var alertingParametersDockerMode = map[string]interface{}{
 	"alertEnabled_BeaconClientSyncComplete":    nil,
 	"alertEnabled_PortConnectivityCheck":       nil,
 	"alertEnabled_LowETHBalance":               nil,
+	"alertEnabled_ObserveModeActive":           nil,
 	"lowETHBalanceThreshold":                   nil,
 }
 

rocketpool-cli/wallet/end-masquerade.go

@@ -51,6 +51,10 @@ func endMasquerade(yes bool) error {
 
 	fmt.Println("Successfully ended masquerade mode.")
 
+	if status.IsObserve {
+		return promptAndRestartObserveDaemons(rp, yes)
+	}
+
 	// Return
 	return nil
 }

rocketpool-cli/wallet/masquerade.go

@@ -53,8 +53,46 @@ func masquerade(addressFlag string, yes bool, observe bool) error {
 
 	fmt.Printf("Your node is now masquerading as address %s.\n", color.LightBlue(address.Hex()))
 	if observe {
-		fmt.Println("Restart the node and watchtower daemons to observe as the masquerade address.")
+		return promptAndRestartObserveDaemons(rp, yes)
 	}
 
 	return nil
 }
+
+// promptAndRestartObserveDaemons asks the user whether to restart the node and watchtower
+// containers now, so they immediately pick up the new observe-mode state.
+func promptAndRestartObserveDaemons(rp *rocketpool.Client, yes bool) error {
+	cfg, _, err := rp.LoadConfig()
+	if err != nil {
+		return fmt.Errorf("error loading configuration: %w", err)
+	}
+
+	if cfg.IsNativeMode {
+		fmt.Println("Restart the node and watchtower daemons for this to take effect.")
+		return nil
+	}
+
+	if !yes && !prompt.Confirm("Would you like to restart the node and watchtower containers now?") {
+		fmt.Println("Remember to restart the node and watchtower containers for this to take effect.")
+		return nil
+	}
+
+	projectName := cfg.Smartnode.ProjectName.Value.(string)
+	for _, name := range []string{"node", "watchtower"} {
+		container := fmt.Sprintf("%s_%s", projectName, name)
+		fmt.Printf("Restarting %s... ", container)
+		response, err := rp.RestartContainer(container)
+		if err != nil {
+			fmt.Println()
+			return fmt.Errorf("error restarting %s: %w", container, err)
+		}
+		if response != container {
+			fmt.Println()
+			return fmt.Errorf("unexpected output while restarting %s: %s", container, response)
+		}
+		fmt.Println("done!")
+	}
+
+	fmt.Println("Done!")
+	return nil
+}

rocketpool/node/node.go

@@ -47,6 +47,7 @@ const (
 	DistributeMinipoolsColor       = color.FgHiGreen
 	ErrorColor                     = color.FgRed
 	WarningColor                   = color.FgYellow
+	ObserveWarningColor            = color.FgHiRed
 	UpdateColor                    = color.FgHiWhite
 	PrestakeMegapoolValidatorColor = color.FgHiGreen
 	StakeMegapoolValidatorColor    = color.FgHiBlue
@@ -176,16 +177,10 @@ func run(c *cli.Command) error {
 		return fmt.Errorf("error getting node account: %w", err)
 	}
 
-	if isObserveMode {
-		red := color.New(color.FgHiRed).SprintFunc()
-		fmt.Println(red("Node daemon is observing address " + nodeAccount.Address.Hex() + "."))
-		fmt.Println(red("Transactions will not be submitted. Fee recipient management targets your real node address."))
-		fmt.Println(red("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing."))
-	}
-
 	// Initialize loggers
 	errorLog := log.NewColorLogger(ErrorColor)
 	updateLog := log.NewColorLogger(UpdateColor)
+	observeLog := log.NewColorLogger(ObserveWarningColor)
 
 	// Create the state provider. In live mode this is a NetworkStateManager
 	// backed by the real EC/BC; in --network-state mode it is a
@@ -348,6 +343,16 @@ func run(c *cli.Command) error {
 			}
 			stateLocker.UpdateState(state)
 
+			// Keep the observe mode warning visible in the logs and the alert active for as long as observe mode is in effect
+			if isObserveMode {
+				observeLog.Println("Node daemon is observing address " + nodeAccount.Address.Hex() + ".")
+				observeLog.Println("Transactions will not be submitted. Fee recipient management targets your real node address.")
+				observeLog.Println("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.")
+				if err := alerting.AlertObserveModeActive(cfg, nodeAccount.Address); err != nil {
+					errorLog.Println(err)
+				}
+			}
+
 			// Manage the fee recipient for the node
 			if err := manageFeeRecipient.run(state); err != nil {
 				errorLog.Println(err)

rocketpool/watchtower/watchtower.go

@@ -21,6 +21,7 @@ import (
 	"github.com/rocket-pool/smartnode/bindings/utils"
 	"github.com/rocket-pool/smartnode/rocketpool/watchtower/collectors"
 	"github.com/rocket-pool/smartnode/shared/services"
+	"github.com/rocket-pool/smartnode/shared/services/alerting"
 	"github.com/rocket-pool/smartnode/shared/services/beacon"
 	"github.com/rocket-pool/smartnode/shared/services/state"
 	"github.com/rocket-pool/smartnode/shared/services/wallet"
@@ -48,6 +49,7 @@ const (
 	MetricsColor                    = color.FgHiYellow
 	SubmitRewardsTreeColor          = color.FgHiCyan
 	WarningColor                    = color.FgYellow
+	ObserveWarningColor             = color.FgHiRed
 	ProcessPenaltiesColor           = color.FgHiMagenta
 	CancelBondsColor                = color.FgGreen
 	CheckSoloMigrationsColor        = color.FgCyan
@@ -158,6 +160,7 @@ func run(c *cli.Command) error {
 	// Initialize error logger
 	errorLog := log.NewColorLogger(ErrorColor)
 	updateLog := log.NewColorLogger(UpdateColor)
+	observeLog := log.NewColorLogger(ObserveWarningColor)
 
 	// Create the state manager
 	m := state.NewNetworkStateManager(rp, cfg.Smartnode.GetStateManagerContracts(), bc, &updateLog)
@@ -168,13 +171,6 @@ func run(c *cli.Command) error {
 		return fmt.Errorf("error getting node account: %w", err)
 	}
 
-	if isObserveMode {
-		red := color.New(color.FgHiRed).SprintFunc()
-		fmt.Println(red("Watchtower daemon is observing address " + nodeAccount.Address.Hex() + "."))
-		fmt.Println(red("Transactions will not be submitted."))
-		fmt.Println(red("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing."))
-	}
-
 	// Initialize tasks
 	respondChallenges, err := newRespondChallenges(c, log.NewColorLogger(RespondChallengesColor), m)
 	if err != nil {
@@ -286,6 +282,16 @@ func run(c *cli.Command) error {
 				continue
 			}
 
+			// Keep the observe mode warning visible in the logs and the alert active for as long as observe mode is in effect
+			if isObserveMode {
+				observeLog.Println("Watchtower daemon is observing address " + nodeAccount.Address.Hex() + ".")
+				observeLog.Println("Transactions will not be submitted.")
+				observeLog.Println("Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.")
+				if err := alerting.AlertObserveModeActive(cfg, nodeAccount.Address); err != nil {
+					errorLog.Println(err)
+				}
+			}
+
 			// Run the manual rewards tree generation
 			if err := generateRewardsTree.run(); err != nil {
 				errorLog.Println(err)

shared/services/alerting/alerting.go

@@ -256,6 +256,31 @@ const (
 	ClientKindBeacon    ClientKind = "Beacon"
 )
 
+// Sends an alert while the node/watchtower daemon is running in observe (masquerade) mode.
+// Called on every task loop iteration for as long as observe mode is active.
+// If alerting/metrics are disabled, this function does nothing.
+func AlertObserveModeActive(cfg *config.RocketPoolConfig, observedAddress common.Address) error {
+	if !isAlertingEnabled(cfg) {
+		logMessage("alerting is disabled, not sending AlertObserveModeActive.")
+		return nil
+	}
+
+	if cfg.Alertmanager.AlertEnabled_ObserveModeActive.Value != true {
+		logMessage("alert for ObserveModeActive is disabled, not sending.")
+		return nil
+	}
+
+	alert := createAlert(
+		fmt.Sprintf("ObserveModeActive-%s", observedAddress.Hex()),
+		"Node is running in observe mode",
+		fmt.Sprintf("The node/watchtower daemon is observing address %s and will not submit transactions. Run `rocketpool wallet end-masquerade` and restart the node/watchtower daemons when you have finished observing.", observedAddress.Hex()),
+		SeverityWarning,
+		strfmt.DateTime(time.Now().Add(DefaultEndsAtDurationForSeverityInfo)),
+		map[string]string{"address": observedAddress.Hex()},
+	)
+	return sendAlert(alert, cfg)
+}
+
 func alertClientSyncComplete(cfg *config.RocketPoolConfig, client ClientKind) error {
 	alertName := fmt.Sprintf("%sClientSyncComplete", client)
 	if !isAlertingEnabled(cfg) {

shared/services/config/alertmanager-config.go

@@ -85,6 +85,8 @@ type AlertmanagerConfig struct {
 	AlertEnabled_BeaconClientSyncComplete    config.Parameter `yaml:"alertEnabled_BeaconClientSyncComplete,omitempty"`
 	// Whether to periodically check if the eth1 and eth2 P2P ports are open to the internet
 	AlertEnabled_PortConnectivityCheck config.Parameter `yaml:"alertEnabled_PortConnectivityCheck,omitempty"`
+	// Whether to alert while the node/watchtower daemon is running in observe (masquerade) mode
+	AlertEnabled_ObserveModeActive config.Parameter `yaml:"alertEnabled_ObserveModeActive,omitempty"`
 }
 
 func NewAlertmanagerConfig(cfg *RocketPoolConfig) *AlertmanagerConfig {
@@ -286,6 +288,10 @@ func NewAlertmanagerConfig(cfg *RocketPoolConfig) *AlertmanagerConfig {
 			"LowETHBalance",
 			"Low ETH Balance"),
 
+		AlertEnabled_ObserveModeActive: createParameterForAlertEnablement(
+			"ObserveModeActive",
+			"the node/watchtower daemon is running in observe mode"),
+
 		LowETHBalanceThreshold: config.Parameter{
 			ID:                 "lowETHBalanceThreshold",
 			Name:               "Low ETH Balance Threshold",
@@ -344,6 +350,7 @@ func (cfg *AlertmanagerConfig) GetParameters() []*config.Parameter {
 		&cfg.AlertEnabled_BeaconClientSyncComplete,
 		&cfg.AlertEnabled_PortConnectivityCheck,
 		&cfg.AlertEnabled_LowETHBalance,
+		&cfg.AlertEnabled_ObserveModeActive,
 		&cfg.LowETHBalanceThreshold,
 	}
 }