From f53800b442b7acb9e6ba1eee4a4ec7cef5b3badd Mon Sep 17 00:00:00 2001 From: Alex Feldgendler Date: Fri, 21 Aug 2026 23:33:36 +0200 Subject: [PATCH] Add separately configurable liveness and readiness probes --- .github/workflows/test.yaml | 6 ++ livekit-server/.helmignore | 2 + livekit-server/templates/deployment.yaml | 12 +-- livekit-server/tests/fixtures/no-probes.yaml | 2 + livekit-server/tests/probes_test.yaml | 98 ++++++++++++++++++++ livekit-server/values.yaml | 27 ++++++ server-sample.yaml | 8 ++ 7 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 livekit-server/tests/fixtures/no-probes.yaml create mode 100644 livekit-server/tests/probes_test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 065370f..a23ec0a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,6 +41,12 @@ jobs: - name: Set up chart-testing uses: helm/chart-testing-action@v2.6.1 + - name: Run chart unit tests + run: |- + # helm 4 requires --verify=false on this source; helm is pinned to 3.x above + helm plugin install https://github.com/helm-unittest/helm-unittest --version v1.0.2 + helm unittest livekit-server + - name: Run chart-testing (lint) run: |- ct lint \ diff --git a/livekit-server/.helmignore b/livekit-server/.helmignore index 0e8a0eb..db8068c 100644 --- a/livekit-server/.helmignore +++ b/livekit-server/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# Chart tests (helm-unittest); not part of the packaged chart +tests/ diff --git a/livekit-server/templates/deployment.yaml b/livekit-server/templates/deployment.yaml index 7b35cee..f76fb05 100644 --- a/livekit-server/templates/deployment.yaml +++ b/livekit-server/templates/deployment.yaml @@ -96,14 +96,14 @@ spec: protocol: UDP {{- end }} {{- end }} + {{- with .Values.livenessProbe }} livenessProbe: - httpGet: - path: / - port: http + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} readinessProbe: - httpGet: - path: / - port: http + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} {{- if or .Values.storeKeysInSecret.enabled (and .Values.livekit.turn.enabled .Values.livekit.turn.tls_port (not .Values.livekit.turn.external_tls)) }} diff --git a/livekit-server/tests/fixtures/no-probes.yaml b/livekit-server/tests/fixtures/no-probes.yaml new file mode 100644 index 0000000..c96c57d --- /dev/null +++ b/livekit-server/tests/fixtures/no-probes.yaml @@ -0,0 +1,2 @@ +livenessProbe: ~ +readinessProbe: ~ diff --git a/livekit-server/tests/probes_test.yaml b/livekit-server/tests/probes_test.yaml new file mode 100644 index 0000000..a4b7044 --- /dev/null +++ b/livekit-server/tests/probes_test.yaml @@ -0,0 +1,98 @@ +suite: liveness and readiness probes +# configmap.yaml is listed because deployment.yaml includes it for its +# checksum annotation; assertions below target deployment.yaml only. +templates: + - deployment.yaml + - configmap.yaml +tests: + - template: deployment.yaml + it: does not make liveness depend on the redis-coupled health endpoint + # regression guard for https://github.com/livekit/livekit/issues/4663: + # `GET /` goes 406 on every node at once during a redis outage, which must + # not be able to restart the fleet + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port + value: http + - notExists: + path: spec.template.spec.containers[0].livenessProbe.httpGet + + - template: deployment.yaml + it: keeps readiness on the health endpoint, so stale nodes leave rotation + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: / + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: http + + - template: deployment.yaml + it: gives the probes independent definitions + asserts: + - notEqual: + path: spec.template.spec.containers[0].livenessProbe + value: + httpGet: + path: / + port: http + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + + - template: deployment.yaml + it: keeps the pre-existing 30s of failures before a liveness restart + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.periodSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].livenessProbe.failureThreshold + value: 3 + + - template: deployment.yaml + it: lets one probe be overridden without disturbing the other + set: + readinessProbe: + httpGet: + path: /readyz + port: http + failureThreshold: 1 + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /readyz + - equal: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port + value: http + + - template: deployment.yaml + it: omits either probe when it is set to null + values: + - ./fixtures/no-probes.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].livenessProbe + - notExists: + path: spec.template.spec.containers[0].readinessProbe + - exists: + path: spec.template.spec.containers[0].image + + - template: deployment.yaml + it: follows a custom livekit.port through the named port + set: + livekit.port: 8080 + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 8080 + - equal: + path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port + value: http + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: http diff --git a/livekit-server/values.yaml b/livekit-server/values.yaml index bcf6d51..6a8f862 100644 --- a/livekit-server/values.yaml +++ b/livekit-server/values.yaml @@ -95,6 +95,33 @@ resources: # cpu: 4000m # memory: 1024Mi +# Liveness and readiness are deliberately different checks: a liveness failure +# restarts the container, a readiness failure only removes the pod from Service +# endpoints. Liveness is a TCP check because `GET /` reports stats staleness, +# which on a redis deployment depends on a keepalive round-tripping through +# redis -- sharing it would restart every pod at once during a redis outage. +# See https://github.com/livekit/livekit/issues/4663. Set to `~` to omit. +livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + +# `GET /` is the right readiness check: it goes non-200 while the node's stats +# are stale, so the node stops taking new sessions until it recovers. Point +# this at a drain-aware endpoint (with failureThreshold: 1) if your server +# build exposes one. +readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + serviceAccount: # Specifies whether a service account should be created create: false diff --git a/server-sample.yaml b/server-sample.yaml index 39ed684..1fa44a2 100644 --- a/server-sample.yaml +++ b/server-sample.yaml @@ -9,6 +9,14 @@ replicaCount: 1 # Suggested value for gracefully terminate the pod: 5 hours terminationGracePeriodSeconds: 18000 +# Probes are configured independently; see values.yaml for the defaults. +# Take the pod out of rotation as soon as it starts draining: +# readinessProbe: +# httpGet: +# path: /readyz +# port: http +# failureThreshold: 1 + livekit: # port: 7880 # Uncomment to enable prometheus metrics