WIP: replicated host catch-up gate rework + design research

.github/workflows/run_tests.yaml

@@ -16,6 +16,27 @@ on:
         type: string
         required: false
 jobs:
+  # Go unit tests were previously run by nothing at all, which is how pkg/model's registry
+  # concurrency test sat broken for two years without anyone noticing. Runs beside the e2e
+  # job, so it costs no extra wall clock. -race is the point: those tests exist to catch
+  # locking regressions and only the detector can see them.
+  go_unit_tests:
+    name: Go unit tests (race)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          # go.mod is the single source of the build Go version.
+          go-version-file: go.mod
+
+      # -vet=off is the project convention: plain `go test` fails to build a few packages
+      # on pre-existing vet noise unrelated to the tests.
+      - name: go test -race
+        run: go test -vet=off -race -count=1 ./pkg/...
+
   run_tests:
     name: Run Tests
     runs-on: ubuntu-latest

config/config.yaml

@@ -502,6 +502,16 @@ reconcile:
         # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
         # is within this specified delay (in seconds)
         delay: 10
+        # Optional replicated-host catch-up gate before advancing to the next host.
+        # Disabled by default to preserve existing reconcile behavior.
+        catchUp:
+          enabled: "false"
+          # Per-host wall-clock budget for the catch-up gate, in seconds.
+          timeout: 900
+          onTimeout: "abort"
+          health:
+            pollInterval: 10
+            successThreshold: 6
       probes:
         # Whether the operator during host launch procedure should wait for startup probe to succeed.
         # In case probe is unspecified wait is assumed to be completed successfully.

deploy/builder/templates-config/config.yaml

@@ -496,6 +496,16 @@ reconcile:
         # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
         # is within this specified delay (in seconds)
         delay: 10
+        # Optional replicated-host catch-up gate before advancing to the next host.
+        # Disabled by default to preserve existing reconcile behavior.
+        catchUp:
+          enabled: "false"
+          # Per-host wall-clock budget for the catch-up gate, in seconds.
+          timeout: 900
+          onTimeout: "abort"
+          health:
+            pollInterval: 10
+            successThreshold: 6
       probes:
         # Whether the operator during host launch procedure should wait for startup probe to succeed.
         # In case probe is unspecified wait is assumed to be completed successfully.

deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml

@@ -447,6 +447,41 @@ spec:
                                 delay:
                                   type: integer
                                   description: "replication max absolute delay to consider replica is not delayed"
+                                catchUp:
+                                  type: object
+                                  description: "Replicated-host catch-up gate to run before advancing to the next host of the shard"
+                                  properties:
+                                    enabled:
+                                      <<: *TypeStringBool
+                                      description: "Whether to run the replicated-host catch-up gate"
+                                    timeout:
+                                      type: integer
+                                      description: "Per-host wall-clock budget for the catch-up gate, in seconds. Omit to use the default"
+                                      minimum: 1
+                                    onTimeout:
+                                      type: string
+                                      description: |
+                                        What to do when the gate does not complete within timeout.
+                                          abort (default) — stop the reconcile
+                                          proceed — advance to the next host without writing the caught-up marker
+                                      enum:
+                                        - ""
+                                        - "Abort"
+                                        - "abort"
+                                        - "Proceed"
+                                        - "proceed"
+                                    health:
+                                      type: object
+                                      description: "Stable-health window required after the host synced its replicated objects"
+                                      properties:
+                                        pollInterval:
+                                          type: integer
+                                          description: "How often to re-check host health, in seconds. Omit to use the default"
+                                          minimum: 1
+                                        successThreshold:
+                                          type: integer
+                                          description: "How many consecutive healthy checks conclude the gate. Omit to use the default"
+                                          minimum: 1
                             probes:
                               type: object
                               description: "What probes the operator should wait during host launch procedure"

deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml

@@ -447,6 +447,41 @@ spec:
                                 delay:
                                   type: integer
                                   description: "replication max absolute delay to consider replica is not delayed"
+                                catchUp:
+                                  type: object
+                                  description: "Replicated-host catch-up gate to run before advancing to the next host of the shard"
+                                  properties:
+                                    enabled:
+                                      !!merge <<: *TypeStringBool
+                                      description: "Whether to run the replicated-host catch-up gate"
+                                    timeout:
+                                      type: integer
+                                      description: "Per-host wall-clock budget for the catch-up gate, in seconds. Omit to use the default"
+                                      minimum: 1
+                                    onTimeout:
+                                      type: string
+                                      description: |
+                                        What to do when the gate does not complete within timeout.
+                                          abort (default) — stop the reconcile
+                                          proceed — advance to the next host without writing the caught-up marker
+                                      enum:
+                                        - ""
+                                        - "Abort"
+                                        - "abort"
+                                        - "Proceed"
+                                        - "proceed"
+                                    health:
+                                      type: object
+                                      description: "Stable-health window required after the host synced its replicated objects"
+                                      properties:
+                                        pollInterval:
+                                          type: integer
+                                          description: "How often to re-check host health, in seconds. Omit to use the default"
+                                          minimum: 1
+                                        successThreshold:
+                                          type: integer
+                                          description: "How many consecutive healthy checks conclude the gate. Omit to use the default"
+                                          minimum: 1
                             probes:
                               type: object
                               description: "What probes the operator should wait during host launch procedure"

deploy/helm/clickhouse-operator/values.yaml

@@ -804,6 +804,16 @@ configs:
               # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
               # is within this specified delay (in seconds)
               delay: 10
+              # Optional replicated-host catch-up gate before advancing to the next host.
+              # Disabled by default to preserve existing reconcile behavior.
+              catchUp:
+                enabled: "false"
+                # Per-host wall-clock budget for the catch-up gate, in seconds.
+                timeout: 900
+                onTimeout: "abort"
+                health:
+                  pollInterval: 10
+                  successThreshold: 6
             probes:
               # Whether the operator during host launch procedure should wait for startup probe to succeed.
               # In case probe is unspecified wait is assumed to be completed successfully.

deploy/operator/clickhouse-operator-install-ansible.yaml

@@ -4160,6 +4160,41 @@ spec:
                                 delay:
                                   type: integer
                                   description: "replication max absolute delay to consider replica is not delayed"
+                                catchUp:
+                                  type: object
+                                  description: "Replicated-host catch-up gate to run before advancing to the next host of the shard"
+                                  properties:
+                                    enabled:
+                                      <<: *TypeStringBool
+                                      description: "Whether to run the replicated-host catch-up gate"
+                                    timeout:
+                                      type: integer
+                                      description: "Per-host wall-clock budget for the catch-up gate, in seconds. Omit to use the default"
+                                      minimum: 1
+                                    onTimeout:
+                                      type: string
+                                      description: |
+                                        What to do when the gate does not complete within timeout.
+                                          abort (default) — stop the reconcile
+                                          proceed — advance to the next host without writing the caught-up marker
+                                      enum:
+                                        - ""
+                                        - "Abort"
+                                        - "abort"
+                                        - "Proceed"
+                                        - "proceed"
+                                    health:
+                                      type: object
+                                      description: "Stable-health window required after the host synced its replicated objects"
+                                      properties:
+                                        pollInterval:
+                                          type: integer
+                                          description: "How often to re-check host health, in seconds. Omit to use the default"
+                                          minimum: 1
+                                        successThreshold:
+                                          type: integer
+                                          description: "How many consecutive healthy checks conclude the gate. Omit to use the default"
+                                          minimum: 1
                             probes:
                               type: object
                               description: "What probes the operator should wait during host launch procedure"
@@ -6193,6 +6228,16 @@ data:
             # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
             # is within this specified delay (in seconds)
             delay: 10
+            # Optional replicated-host catch-up gate before advancing to the next host.
+            # Disabled by default to preserve existing reconcile behavior.
+            catchUp:
+              enabled: "false"
+              # Per-host wall-clock budget for the catch-up gate, in seconds.
+              timeout: 900
+              onTimeout: "abort"
+              health:
+                pollInterval: 10
+                successThreshold: 6
           probes:
             # Whether the operator during host launch procedure should wait for startup probe to succeed.
             # In case probe is unspecified wait is assumed to be completed successfully.

deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml

@@ -4127,6 +4127,41 @@ spec:
                             delay:
                               type: integer
                               description: "replication max absolute delay to consider replica is not delayed"
+                            catchUp:
+                              type: object
+                              description: "Replicated-host catch-up gate to run before advancing to the next host of the shard"
+                              properties:
+                                enabled:
+                                  !!merge <<: *TypeStringBool
+                                  description: "Whether to run the replicated-host catch-up gate"
+                                timeout:
+                                  type: integer
+                                  description: "Per-host wall-clock budget for the catch-up gate, in seconds. Omit to use the default"
+                                  minimum: 1
+                                onTimeout:
+                                  type: string
+                                  description: |
+                                    What to do when the gate does not complete within timeout.
+                                      abort (default) — stop the reconcile
+                                      proceed — advance to the next host without writing the caught-up marker
+                                  enum:
+                                    - ""
+                                    - "Abort"
+                                    - "abort"
+                                    - "Proceed"
+                                    - "proceed"
+                                health:
+                                  type: object
+                                  description: "Stable-health window required after the host synced its replicated objects"
+                                  properties:
+                                    pollInterval:
+                                      type: integer
+                                      description: "How often to re-check host health, in seconds. Omit to use the default"
+                                      minimum: 1
+                                    successThreshold:
+                                      type: integer
+                                      description: "How many consecutive healthy checks conclude the gate. Omit to use the default"
+                                      minimum: 1
                         probes:
                           type: object
                           description: "What probes the operator should wait during host launch procedure"
@@ -6392,6 +6427,16 @@ data:
             # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
             # is within this specified delay (in seconds)
             delay: 10
+            # Optional replicated-host catch-up gate before advancing to the next host.
+            # Disabled by default to preserve existing reconcile behavior.
+            catchUp:
+              enabled: "false"
+              # Per-host wall-clock budget for the catch-up gate, in seconds.
+              timeout: 900
+              onTimeout: "abort"
+              health:
+                pollInterval: 10
+                successThreshold: 6
           probes:
             # Whether the operator during host launch procedure should wait for startup probe to succeed.
             # In case probe is unspecified wait is assumed to be completed successfully.

deploy/operator/clickhouse-operator-install-bundle.yaml

@@ -4153,6 +4153,41 @@ spec:
                                 delay:
                                   type: integer
                                   description: "replication max absolute delay to consider replica is not delayed"
+                                catchUp:
+                                  type: object
+                                  description: "Replicated-host catch-up gate to run before advancing to the next host of the shard"
+                                  properties:
+                                    enabled:
+                                      <<: *TypeStringBool
+                                      description: "Whether to run the replicated-host catch-up gate"
+                                    timeout:
+                                      type: integer
+                                      description: "Per-host wall-clock budget for the catch-up gate, in seconds. Omit to use the default"
+                                      minimum: 1
+                                    onTimeout:
+                                      type: string
+                                      description: |
+                                        What to do when the gate does not complete within timeout.
+                                          abort (default) — stop the reconcile
+                                          proceed — advance to the next host without writing the caught-up marker
+                                      enum:
+                                        - ""
+                                        - "Abort"
+                                        - "abort"
+                                        - "Proceed"
+                                        - "proceed"
+                                    health:
+                                      type: object
+                                      description: "Stable-health window required after the host synced its replicated objects"
+                                      properties:
+                                        pollInterval:
+                                          type: integer
+                                          description: "How often to re-check host health, in seconds. Omit to use the default"
+                                          minimum: 1
+                                        successThreshold:
+                                          type: integer
+                                          description: "How many consecutive healthy checks conclude the gate. Omit to use the default"
+                                          minimum: 1
                             probes:
                               type: object
                               description: "What probes the operator should wait during host launch procedure"
@@ -6452,6 +6487,16 @@ data:
             # a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"
             # is within this specified delay (in seconds)
             delay: 10
+            # Optional replicated-host catch-up gate before advancing to the next host.
+            # Disabled by default to preserve existing reconcile behavior.
+            catchUp:
+              enabled: "false"
+              # Per-host wall-clock budget for the catch-up gate, in seconds.
+              timeout: 900
+              onTimeout: "abort"
+              health:
+                pollInterval: 10
+                successThreshold: 6
           probes:
             # Whether the operator during host launch procedure should wait for startup probe to succeed.
             # In case probe is unspecified wait is assumed to be completed successfully.