Add separately configurable liveness and readiness probes

.github/workflows/test.yaml

@@ -41,6 +41,12 @@ jobs:
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.6.1
 
+      - name: Run chart unit tests
+        run: |-
+          # helm 4 requires --verify=false on this source; helm is pinned to 3.x above
+          helm plugin install https://github.com/helm-unittest/helm-unittest --version v1.0.2
+          helm unittest livekit-server
+
       - name: Run chart-testing (lint)
         run: |-
           ct lint \

livekit-server/.helmignore

@@ -21,3 +21,5 @@
 .idea/
 *.tmproj
 .vscode/
+# Chart tests (helm-unittest); not part of the packaged chart
+tests/

livekit-server/templates/deployment.yaml

@@ -96,14 +96,14 @@ spec:
               protocol: UDP
             {{- end }}
             {{- end }}
+          {{- with .Values.livenessProbe }}
           livenessProbe:
-            httpGet:
-              path: /
-              port: http
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
           readinessProbe:
-            httpGet:
-              path: /
-              port: http
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           {{- if or .Values.storeKeysInSecret.enabled (and .Values.livekit.turn.enabled .Values.livekit.turn.tls_port (not .Values.livekit.turn.external_tls)) }}

livekit-server/tests/fixtures/no-probes.yaml

@@ -0,0 +1,2 @@
+livenessProbe: ~
+readinessProbe: ~

livekit-server/tests/probes_test.yaml

@@ -0,0 +1,98 @@
+suite: liveness and readiness probes
+# configmap.yaml is listed because deployment.yaml includes it for its
+# checksum annotation; assertions below target deployment.yaml only.
+templates:
+  - deployment.yaml
+  - configmap.yaml
+tests:
+  - template: deployment.yaml
+    it: does not make liveness depend on the redis-coupled health endpoint
+    # regression guard for https://github.com/livekit/livekit/issues/4663:
+    # `GET /` goes 406 on every node at once during a redis outage, which must
+    # not be able to restart the fleet
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port
+          value: http
+      - notExists:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet
+
+  - template: deployment.yaml
+    it: keeps readiness on the health endpoint, so stale nodes leave rotation
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+          value: http
+
+  - template: deployment.yaml
+    it: gives the probes independent definitions
+    asserts:
+      - notEqual:
+          path: spec.template.spec.containers[0].livenessProbe
+          value:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 0
+            periodSeconds: 10
+            timeoutSeconds: 1
+            failureThreshold: 3
+
+  - template: deployment.yaml
+    it: keeps the pre-existing 30s of failures before a liveness restart
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 3
+
+  - template: deployment.yaml
+    it: lets one probe be overridden without disturbing the other
+    set:
+      readinessProbe:
+        httpGet:
+          path: /readyz
+          port: http
+        failureThreshold: 1
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /readyz
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port
+          value: http
+
+  - template: deployment.yaml
+    it: omits either probe when it is set to null
+    values:
+      - ./fixtures/no-probes.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.containers[0].livenessProbe
+      - notExists:
+          path: spec.template.spec.containers[0].readinessProbe
+      - exists:
+          path: spec.template.spec.containers[0].image
+
+  - template: deployment.yaml
+    it: follows a custom livekit.port through the named port
+    set:
+      livekit.port: 8080
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].ports[0].containerPort
+          value: 8080
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port
+          value: http
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.port
+          value: http

livekit-server/values.yaml

@@ -95,6 +95,33 @@ resources:
   #   cpu: 4000m
   #   memory: 1024Mi
 
+# Liveness and readiness are deliberately different checks: a liveness failure
+# restarts the container, a readiness failure only removes the pod from Service
+# endpoints. Liveness is a TCP check because `GET /` reports stats staleness,
+# which on a redis deployment depends on a keepalive round-tripping through
+# redis -- sharing it would restart every pod at once during a redis outage.
+# See https://github.com/livekit/livekit/issues/4663. Set to `~` to omit.
+livenessProbe:
+  tcpSocket:
+    port: http
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  timeoutSeconds: 1
+  failureThreshold: 3
+
+# `GET /` is the right readiness check: it goes non-200 while the node's stats
+# are stale, so the node stops taking new sessions until it recovers. Point
+# this at a drain-aware endpoint (with failureThreshold: 1) if your server
+# build exposes one.
+readinessProbe:
+  httpGet:
+    path: /
+    port: http
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  failureThreshold: 3
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: false

server-sample.yaml

@@ -9,6 +9,14 @@ replicaCount: 1
 # Suggested value for gracefully terminate the pod: 5 hours
 terminationGracePeriodSeconds: 18000
 
+# Probes are configured independently; see values.yaml for the defaults.
+# Take the pod out of rotation as soon as it starts draining:
+# readinessProbe:
+#   httpGet:
+#     path: /readyz
+#     port: http
+#   failureThreshold: 1
+
 livekit:
   # port: 7880
   # Uncomment to enable prometheus metrics