Protect CHK recreate with live Raft quorum safety

pkg/apis/clickhouse-keeper.altinity.com/v1/type_status.go

@@ -48,6 +48,7 @@ const (
 	StatusReasonFIPSValidationFailed     = chi.StatusReasonFIPSValidationFailed
 	StatusReasonFIPSImagePolicyViolation = chi.StatusReasonFIPSImagePolicyViolation
 	StatusReasonNoKeeperListener         = chi.StatusReasonNoKeeperListener
+	StatusReasonRaftQuorumUnsafe         = chi.StatusReasonRaftQuorumUnsafe
 )
 
 // Status defines status section of the custom resource.

pkg/apis/clickhouse.altinity.com/v1/type_status.go

@@ -69,6 +69,9 @@ const (
 	// the shard left serving. Recoverable without a spec edit - the peer coming back triggers
 	// a retry - so it is deliberately absent from normalizeTimeAbortReasons.
 	StatusReasonShardHasNoHealthyPeer = "ShardHasNoHealthyPeer"
+	// StatusReasonRaftQuorumUnsafe: a CHK host's reconcile would disrupt a Ready replica while
+	// the ensemble lacks Raft quorum headroom. Disruption is deferred until siblings recover.
+	StatusReasonRaftQuorumUnsafe = "RaftQuorumUnsafe"
 	// StatusReasonRemovedSecretRefSyntax: a user settings field uses the `k8s_secret_` or
 	// `k8s_secret_env_` prefix, removed in 0.27.4 because it accepted a namespace/name/key
 	// triple and could therefore read a Secret from any namespace. Aborts rather than

pkg/controller/chk/controller.go

@@ -16,6 +16,7 @@ package chk
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	apiExtensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -30,11 +31,16 @@ import (
 	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
 	"github.com/altinity/clickhouse-operator/pkg/chop"
 	"github.com/altinity/clickhouse-operator/pkg/controller/chk/kube"
+	"github.com/altinity/clickhouse-operator/pkg/controller/common"
 	"github.com/altinity/clickhouse-operator/pkg/interfaces"
 	"github.com/altinity/clickhouse-operator/pkg/model/managers"
 	"github.com/altinity/clickhouse-operator/pkg/util"
 )
 
+// raftQuorumDeferredRequeueAfter is the fixed retry interval when reconcile
+// returns ErrCRUDDeferred (Raft quorum headroom not available yet).
+const raftQuorumDeferredRequeueAfter = 5 * time.Second
+
 // Controller reconciles a ClickHouseKeeper object
 type Controller struct {
 	client.Client
@@ -112,7 +118,18 @@ func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil
 	}
 
-	w.reconcileCR(ctx, nil, new)
+	if err := w.reconcileCR(ctx, nil, new); err != nil {
+		if errors.Is(err, common.ErrCRUDDeferred) {
+			// Soft defer: quorum headroom not available yet. Status already
+			// records [RaftQuorumUnsafe]; retry on a fixed interval instead of
+			// error backoff.
+			log.V(1).M(new).F().Info(
+				"Raft quorum defer — requeue in %s", raftQuorumDeferredRequeueAfter,
+			)
+			return ctrl.Result{RequeueAfter: raftQuorumDeferredRequeueAfter}, nil
+		}
+		return ctrl.Result{}, err
+	}
 
 	return ctrl.Result{}, nil
 }

pkg/controller/chk/worker-deleter.go

@@ -16,9 +16,10 @@ package chk
 
 import (
 	"context"
+	"time"
+
 	apiChk "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1"
 	"github.com/altinity/clickhouse-operator/pkg/controller"
-	"time"
 
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -51,6 +52,9 @@ func (w *worker) clean(ctx context.Context, cr api.ICustomResource) {
 	objs.Subtract(need)
 	w.a.V(1).M(cr).F().Info("List of non-reconciled objects:\n%s", objs)
 	if w.purge(ctx, cr, objs, w.task.RegistryFailed()) > 0 {
+		// Give Raft time to notice removed peers before Completed. Survivors
+		// often flip /ready → 503 briefly after purge.
+		w.a.V(1).M(cr).F().Info("Purged non-reconciled objects; waiting 1m for membership to settle")
 		util.WaitContextDoneOrTimeout(ctx, 1*time.Minute)
 	}