diff --git a/controller/Containerfile b/controller/Containerfile index 846e4d340..d79593f21 100644 --- a/controller/Containerfile +++ b/controller/Containerfile @@ -48,11 +48,18 @@ RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1 go build -a \ -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ -o router ./cmd/router +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o telemetry ./cmd/telemetry FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1786321990@sha256:7e7f79ab747bf2b452e3043dd89f388e92be4c7fdcc8b815b58adf6c99c39c95 WORKDIR / COPY --from=builder /build/manager . COPY --from=builder /build/router . +COPY --from=builder /build/telemetry . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/controller/Makefile b/controller/Makefile index 2d64ed9e0..efda65347 100644 --- a/controller/Makefile +++ b/controller/Makefile @@ -123,6 +123,7 @@ build-operator-ci: build: manifests generate fmt vet ## Build manager binary. go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go go build -ldflags "$(LDFLAGS)" -o bin/router ./cmd/router + go build -ldflags "$(LDFLAGS)" -o bin/telemetry ./cmd/telemetry go build -ldflags "$(LDFLAGS)" -o bin/exporter-set-controller cmd/exporter-set-controller/main.go .PHONY: run @@ -149,6 +150,7 @@ docker-build-ci: ## Build docker images from pre-compiled host binaries (fast CI rm -rf bin/ci-stage && mkdir -p bin/ci-stage/controller bin/ci-stage/esc CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/manager cmd/main.go CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/router ./cmd/router + CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/telemetry ./cmd/telemetry CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/esc/exporter-set-controller cmd/exporter-set-controller/main.go $(CONTAINER_TOOL) build --build-arg BIN=manager -t $(IMG) -f Containerfile.prebuilt bin/ci-stage/controller $(CONTAINER_TOOL) build --build-arg BIN=exporter-set-controller -t $(EXPORTER_SET_CONTROLLER_IMG) -f Containerfile.prebuilt bin/ci-stage/esc diff --git a/controller/cmd/telemetry/main.go b/controller/cmd/telemetry/main.go index 1c757d42e..702a8744d 100644 --- a/controller/cmd/telemetry/main.go +++ b/controller/cmd/telemetry/main.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// jumpstarter-telemetry receives structured log entries from exporters and clients -// via the PushLogs gRPC RPC and writes them to structured stdout for downstream -// log shippers (Promtail, Grafana Alloy, Vector) to forward to Loki. +// jumpstarter-telemetry reverse-scrapes exporter metrics via MetricsStream and +// receives structured log entries via PushLogs. Logs are written to structured +// stdout for downstream log shippers (Promtail, Grafana Alloy, Vector) and +// optionally pushed to Loki's HTTP API when -loki-url is set. // // TLS: always enabled. Set EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM to file paths of // operator-mounted cert/key (e.g. from a cert-manager Secret); when absent a @@ -30,8 +31,7 @@ limitations under the License. // certificate; the controller uses it to advertise the address to exporters via // GetServiceEndpoints. A mismatch causes TLS hostname verification failures. // -// Future phases will add direct Loki push and MetricsStream for reverse-scrape -// of exporter prometheus_client registries. +// HTTP: GET /metrics, /healthz, and /readyz bind separately (default :8080). package main import ( @@ -39,7 +39,9 @@ import ( "flag" "os" "os/signal" + "strings" "syscall" + "time" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -55,9 +57,48 @@ var ( buildDate = "unknown" ) +func splitCSV(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + func main() { var bindAddr string + var metricsAddr string + var scrapeTimeout time.Duration + var driverTypeEnum string + var exemplarKeys string + var lokiURL string + var lokiQueueDepth int + var lokiInsecure bool + var lokiCAFile string flag.StringVar(&bindAddr, "grpc-bind", ":9093", "TCP address to bind the gRPC server to") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", + "TCP address for HTTP GET /metrics, /healthz, and /readyz. Use 0 to disable.") + flag.DurationVar(&scrapeTimeout, "scrape-timeout", 7*time.Second, + "Max wait for parallel MetricsStream scrape responses") + flag.StringVar(&driverTypeEnum, "driver-type-enum", strings.Join(service.DefaultDriverTypeEnum, ","), + "Comma-separated allowlist of driver_type values; others are remapped to other") + flag.StringVar(&exemplarKeys, "exemplar-keys", strings.Join(service.DefaultExemplarKeys, ","), + "Comma-separated allowlist of Prometheus exemplar keys") + flag.StringVar(&lokiURL, "loki-url", "", + "Loki HTTP push URL (optional; telemetry runs metrics-only when empty)") + flag.IntVar(&lokiQueueDepth, "loki-queue-depth", 10000, + "Ring buffer depth for Loki log push") + flag.BoolVar(&lokiInsecure, "loki-insecure-skip-verify", false, + "Disable TLS certificate verification for Loki (development/testing only)") + flag.StringVar(&lokiCAFile, "loki-ca-file", "", + "PEM CA bundle used to verify the Loki TLS endpoint") opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -71,6 +112,8 @@ func main() { "gitCommit", gitCommit, "buildDate", buildDate, "bindAddr", bindAddr, + "metricsBindAddr", metricsAddr, + "scrapeTimeout", scrapeTimeout, ) ctx, cancel := context.WithCancel(context.Background()) @@ -87,8 +130,24 @@ func main() { } svc := &service.TelemetryService{ - BindAddr: bindAddr, - Signer: signer, + BindAddr: bindAddr, + MetricsBindAddr: metricsAddr, + ScrapeTimeout: scrapeTimeout, + DriverTypeEnum: splitCSV(driverTypeEnum), + ExemplarKeys: splitCSV(exemplarKeys), + Signer: signer, + LokiConfig: service.LokiConfig{ + URL: lokiURL, + Username: os.Getenv("LOKI_USERNAME"), + Password: os.Getenv("LOKI_PASSWORD"), + Token: os.Getenv("LOKI_TOKEN"), + CAFile: lokiCAFile, + InsecureSkipVerify: lokiInsecure, + QueueDepth: lokiQueueDepth, + }, + } + if lokiURL != "" { + logger.Info("Loki HTTP push configured", "url", lokiURL, "queueDepth", lokiQueueDepth) } // Register signal handler before starting the service so no signal diff --git a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go index ae1271d1f..c9198471f 100644 --- a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -312,6 +312,50 @@ type TelemetryConfig struct { // gRPC configuration for the telemetry service. // Use this to configure TLS when not using cert-manager. GRPC TelemetryGRPCConfig `json:"grpc,omitempty"` + + // Metrics configures reverse-scrape fan-out and Prometheus exposition + // (JEP-0013). ServiceMonitor fields are a later phase. + Metrics TelemetryMetricsConfig `json:"metrics,omitempty"` + + // Loki configures optional HTTP push of ingested logs to a Loki-compatible + // endpoint. When url is empty, telemetry runs metrics-only. + Loki TelemetryLokiConfig `json:"loki,omitempty"` + + // Backpressure configures the Loki log push ring buffer. + Backpressure TelemetryBackpressureConfig `json:"backpressure,omitempty"` +} + +// TelemetryLokiConfig configures Loki HTTP push from the telemetry service. +type TelemetryLokiConfig struct { + // Loki push endpoint (http:// or https://). Optional — telemetry can run + // metrics-only without Loki. grpc:// is reserved but not implemented yet. + URL string `json:"url,omitempty"` + + // Secret with Loki credentials (username/password and/or token keys). + // See JEP-0013 DD-5: only the telemetry pod holds Loki credentials. + SecretRef string `json:"secretRef,omitempty"` + + // TLS settings for the Loki endpoint. + TLS TelemetryLokiTLSConfig `json:"tls,omitempty"` +} + +// TelemetryLokiTLSConfig configures TLS for the Loki push endpoint. +type TelemetryLokiTLSConfig struct { + // Secret containing a CA bundle (ca.crt key) to trust for the Loki endpoint. + CASecretRef string `json:"caSecretRef,omitempty"` + + // Disable TLS certificate verification (development/testing only). + // +kubebuilder:default=false + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` +} + +// TelemetryBackpressureConfig configures the Loki log push ring buffer. +type TelemetryBackpressureConfig struct { + // Ring buffer depth for Loki log push. On overflow, dropped entries are + // replaced by a single drop-marker LogEntry. + // +kubebuilder:default=10000 + // +kubebuilder:validation:Minimum=1 + QueueDepth int32 `json:"queueDepth,omitempty"` } // TelemetryGRPCConfig defines gRPC configuration for the telemetry service. @@ -325,6 +369,22 @@ type TelemetryGRPCConfig struct { TLS TLSConfig `json:"tls,omitempty"` } +// TelemetryMetricsConfig configures telemetry /metrics reverse-scrape behavior. +type TelemetryMetricsConfig struct { + // Allowlist of keys to include in Prometheus exemplars. Unlisted keys are omitted. + // +kubebuilder:default={"client","lease_id"} + ExemplarKeys []string `json:"exemplarKeys,omitempty"` + + // Allowed driver_type label values. Unlisted types are remapped to "other". + // +kubebuilder:default={"power","storage","network","serial","console","video","composite"} + DriverTypeEnum []string `json:"driverTypeEnum,omitempty"` + + // Max wait for parallel exporter MetricsStream responses during a /metrics fan-out. + // Should be lower than the Prometheus scrape_timeout. + // +kubebuilder:default="7s" + ScrapeTimeout *metav1.Duration `json:"scrapeTimeout,omitempty"` +} + // TelemetryLoggingConfig configures the log push path to the telemetry service. type TelemetryLoggingConfig struct { // Filter controls which log entries are forwarded to the telemetry service. diff --git a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 56f9f7d77..d28d4d6ea 100644 --- a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -891,6 +891,21 @@ func (in *TLSConfig) DeepCopy() *TLSConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryBackpressureConfig) DeepCopyInto(out *TelemetryBackpressureConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryBackpressureConfig. +func (in *TelemetryBackpressureConfig) DeepCopy() *TelemetryBackpressureConfig { + if in == nil { + return nil + } + out := new(TelemetryBackpressureConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TelemetryConfig) DeepCopyInto(out *TelemetryConfig) { *out = *in @@ -902,6 +917,9 @@ func (in *TelemetryConfig) DeepCopyInto(out *TelemetryConfig) { in.Resources.DeepCopyInto(&out.Resources) out.Logging = in.Logging out.GRPC = in.GRPC + in.Metrics.DeepCopyInto(&out.Metrics) + out.Loki = in.Loki + out.Backpressure = in.Backpressure } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryConfig. @@ -960,3 +978,64 @@ func (in *TelemetryLoggingFilterConfig) DeepCopy() *TelemetryLoggingFilterConfig in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryLokiConfig) DeepCopyInto(out *TelemetryLokiConfig) { + *out = *in + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryLokiConfig. +func (in *TelemetryLokiConfig) DeepCopy() *TelemetryLokiConfig { + if in == nil { + return nil + } + out := new(TelemetryLokiConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryLokiTLSConfig) DeepCopyInto(out *TelemetryLokiTLSConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryLokiTLSConfig. +func (in *TelemetryLokiTLSConfig) DeepCopy() *TelemetryLokiTLSConfig { + if in == nil { + return nil + } + out := new(TelemetryLokiTLSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryMetricsConfig) DeepCopyInto(out *TelemetryMetricsConfig) { + *out = *in + if in.ExemplarKeys != nil { + in, out := &in.ExemplarKeys, &out.ExemplarKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DriverTypeEnum != nil { + in, out := &in.DriverTypeEnum, &out.DriverTypeEnum + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ScrapeTimeout != nil { + in, out := &in.ScrapeTimeout, &out.ScrapeTimeout + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryMetricsConfig. +func (in *TelemetryMetricsConfig) DeepCopy() *TelemetryMetricsConfig { + if in == nil { + return nil + } + out := new(TelemetryMetricsConfig) + in.DeepCopyInto(out) + return out +} diff --git a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index f5784b859..1b4b7ccc2 100644 --- a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.21.0 name: jumpstarters.operator.jumpstarter.dev spec: group: operator.jumpstarter.dev @@ -1087,7 +1087,8 @@ spec: Number of controller replicas to run. Currently only 1 replica is supported because the controller uses in-memory state for gRPC stream coordination (Dial/Listen). Values greater than 1 will - be clamped to 1 with a warning. See issue 1013 for HA controller support. + be clamped to 1 with a warning. See https://github.com/jumpstarter-dev/jumpstarter/issues/1013 + for the tracking issue on HA controller support. format: int32 minimum: 1 type: integer @@ -2099,6 +2100,18 @@ spec: structured log entries from exporters via gRPC. The controller advertises the telemetry endpoint to exporters so they can push logs without cluster credentials. properties: + backpressure: + description: Backpressure configures the Loki log push ring buffer. + properties: + queueDepth: + default: 10000 + description: |- + Ring buffer depth for Loki log push. On overflow, dropped entries are + replaced by a single drop-marker LogEntry. + format: int32 + minimum: 1 + type: integer + type: object enabled: default: false description: |- @@ -2163,6 +2176,70 @@ spec: type: string type: object type: object + loki: + description: |- + Loki configures optional HTTP push of ingested logs to a Loki-compatible + endpoint. When url is empty, telemetry runs metrics-only. + properties: + secretRef: + description: |- + Secret with Loki credentials (username/password and/or token keys). + See JEP-0013 DD-5: only the telemetry pod holds Loki credentials. + type: string + tls: + description: TLS settings for the Loki endpoint. + properties: + caSecretRef: + description: Secret containing a CA bundle (ca.crt key) + to trust for the Loki endpoint. + type: string + insecureSkipVerify: + default: false + description: Disable TLS certificate verification (development/testing + only). + type: boolean + type: object + url: + description: |- + Loki push endpoint (http:// or https://). Optional — telemetry can run + metrics-only without Loki. grpc:// is reserved but not implemented yet. + type: string + type: object + metrics: + description: |- + Metrics configures reverse-scrape fan-out and Prometheus exposition + (JEP-0013). ServiceMonitor fields are a later phase. + properties: + driverTypeEnum: + default: + - power + - storage + - network + - serial + - console + - video + - composite + description: Allowed driver_type label values. Unlisted types + are remapped to "other". + items: + type: string + type: array + exemplarKeys: + default: + - client + - lease_id + description: Allowlist of keys to include in Prometheus exemplars. + Unlisted keys are omitted. + items: + type: string + type: array + scrapeTimeout: + default: 7s + description: |- + Max wait for parallel exporter MetricsStream responses during a /metrics fan-out. + Should be lower than the Prometheus scrape_timeout. + type: string + type: object replicas: default: 1 description: |- diff --git a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 1a15ddf47..54899c9e8 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -1326,7 +1326,8 @@ func (r *JumpstarterReconciler) buildConfig(ctx context.Context, jumpstarter *op Keys: jumpstarter.Spec.DeprecatedLabels.Keys, } - // Telemetry configuration. + // Telemetry configuration. When cert-manager is enabled, inline the CA so + // GetServiceEndpoints.certificate lets exporters verify telemetry TLS. if jumpstarter.Spec.Telemetry != nil && jumpstarter.Spec.Telemetry.Enabled { t := jumpstarter.Spec.Telemetry telemetryCfg := &config.Telemetry{ diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go index c94945b17..e06d3b21d 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go @@ -19,6 +19,7 @@ package jumpstarter import ( "context" "fmt" + "strings" "time" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -39,14 +40,27 @@ import ( const ( telemetryPort = 9093 + telemetryMetricsPort = 8080 telemetryCertSuffix = "-telemetry-tls" telemetryServiceName = "jumpstarter-telemetry" telemetryComponentApp = "jumpstarter-telemetry" telemetrySASuffix = "-telemetry" grpcPortName = "grpc" + metricsPortName = "metrics" telemetryCARequeueInterval = 30 * time.Second ) +// JEP-0013 default allowlists used when spec.telemetry.metrics omits a field. +var ( + defaultTelemetryDriverTypeEnum = []string{"power", "storage", "network", "serial", "console", "video", "composite"} + defaultTelemetryExemplarKeys = []string{"client", "lease_id"} + defaultTelemetryScrapeTimeout = 7 * time.Second + defaultTelemetryLokiQueueDepth = int32(10000) + lokiCAMountPath = "/loki-ca" + lokiCAFilePath = "/loki-ca/ca.crt" + lokiCAVolumeName = "loki-ca" +) + // reconcileTelemetryDeploymentStage reconciles only the telemetry Deployment (and cleanup). // It is called from the Deployment stage of the reconcile loop, before Services. // Certificate reconciliation is handled in certificates.go alongside other certs. @@ -251,6 +265,12 @@ func (r *JumpstarterReconciler) reconcileTelemetryService(ctx context.Context, j TargetPort: intstr.FromInt(telemetryPort), Protocol: corev1.ProtocolTCP, }, + { + Name: metricsPortName, + Port: int32(telemetryMetricsPort), + TargetPort: intstr.FromInt(telemetryMetricsPort), + Protocol: corev1.ProtocolTCP, + }, }, }, } @@ -293,6 +313,44 @@ func (r *JumpstarterReconciler) reconcileTelemetryService(ctx context.Context, j return nil } +// telemetryContainerArgs returns gRPC and HTTP flags for the telemetry binary. +func telemetryContainerArgs(t *operatorv1alpha1.TelemetryConfig) []string { + timeout := defaultTelemetryScrapeTimeout + if t.Metrics.ScrapeTimeout != nil && t.Metrics.ScrapeTimeout.Duration > 0 { + timeout = t.Metrics.ScrapeTimeout.Duration + } + enum := t.Metrics.DriverTypeEnum + if len(enum) == 0 { + enum = defaultTelemetryDriverTypeEnum + } + keys := t.Metrics.ExemplarKeys + if len(keys) == 0 { + keys = defaultTelemetryExemplarKeys + } + args := []string{ + fmt.Sprintf("--grpc-bind=:%d", telemetryPort), + fmt.Sprintf("-metrics-bind-address=:%d", telemetryMetricsPort), + fmt.Sprintf("-scrape-timeout=%s", timeout), + fmt.Sprintf("-driver-type-enum=%s", strings.Join(enum, ",")), + fmt.Sprintf("-exemplar-keys=%s", strings.Join(keys, ",")), + } + if t.Loki.URL != "" { + args = append(args, fmt.Sprintf("-loki-url=%s", t.Loki.URL)) + depth := t.Backpressure.QueueDepth + if depth <= 0 { + depth = defaultTelemetryLokiQueueDepth + } + args = append(args, fmt.Sprintf("-loki-queue-depth=%d", depth)) + if t.Loki.TLS.InsecureSkipVerify { + args = append(args, "-loki-insecure-skip-verify=true") + } + if t.Loki.TLS.CASecretRef != "" { + args = append(args, fmt.Sprintf("-loki-ca-file=%s", lokiCAFilePath)) + } + } + return args +} + // createTelemetryDeployment builds the desired Deployment for the telemetry service. // tlsSecretHash is included as a pod annotation to trigger rolling restarts on cert renewal. func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSecretHash string) *appsv1.Deployment { @@ -325,6 +383,10 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec }, }, }, + { + Name: "GRPC_TELEMETRY_ENDPOINT", + Value: telemetryEndpointFor(jumpstarter.Namespace), + }, } var volumeMounts []corev1.VolumeMount @@ -355,6 +417,60 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec }) } + if t.Loki.URL != "" && t.Loki.SecretRef != "" { + optional := ptr.To(true) + envVars = append(envVars, + corev1.EnvVar{ + Name: "LOKI_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: t.Loki.SecretRef}, + Key: "username", + Optional: optional, + }, + }, + }, + corev1.EnvVar{ + Name: "LOKI_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: t.Loki.SecretRef}, + Key: "password", + Optional: optional, + }, + }, + }, + corev1.EnvVar{ + Name: "LOKI_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: t.Loki.SecretRef}, + Key: "token", + Optional: optional, + }, + }, + }, + ) + } + + if t.Loki.URL != "" && t.Loki.TLS.CASecretRef != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: lokiCAVolumeName, + MountPath: lokiCAMountPath, + ReadOnly: true, + }) + defaultMode := int32(420) + volumes = append(volumes, corev1.Volume{ + Name: lokiCAVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: t.Loki.TLS.CASecretRef, + DefaultMode: &defaultMode, + }, + }, + }) + } + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-telemetry", jumpstarter.Name), @@ -390,22 +506,27 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec Image: t.Image, ImagePullPolicy: t.ImagePullPolicy, Command: []string{"/telemetry"}, - Args: []string{ - fmt.Sprintf("--grpc-bind=:%d", telemetryPort), - }, - Env: envVars, - VolumeMounts: volumeMounts, + Args: telemetryContainerArgs(t), + Env: envVars, + VolumeMounts: volumeMounts, Ports: []corev1.ContainerPort{ { ContainerPort: int32(telemetryPort), Name: grpcPortName, Protocol: corev1.ProtocolTCP, }, + { + ContainerPort: int32(telemetryMetricsPort), + Name: metricsPortName, + Protocol: corev1.ProtocolTCP, + }, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(telemetryPort), + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(telemetryMetricsPort), + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: 10, @@ -416,8 +537,10 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(telemetryPort), + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(telemetryMetricsPort), + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: 5, diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go new file mode 100644 index 000000000..65da7d4a8 --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +const ( + jepDefaultLokiQueueDepthFlag = "-loki-queue-depth=10000" + jepLokiCAFileFlag = "-loki-ca-file=/loki-ca/ca.crt" +) + +var _ = Describe("createTelemetryDeployment JEP-0013 Loki push", func() { + It("omits Loki flags when spec.telemetry.loki.url is unset (metrics-only)", func() { + c := createTelemetryDeployment(phase3TelemetryJS("js", "ns"), "").Spec.Template.Spec.Containers[0] + for _, arg := range c.Args { + Expect(arg).NotTo(HavePrefix("-loki-url=")) + Expect(arg).NotTo(HavePrefix("-loki-queue-depth=")) + Expect(arg).NotTo(HavePrefix("-loki-ca-file=")) + Expect(arg).NotTo(HavePrefix("-loki-insecure-skip-verify")) + } + Expect(hasEnv(c, "LOKI_USERNAME")).To(BeFalse()) + Expect(hasEnv(c, "LOKI_PASSWORD")).To(BeFalse()) + Expect(hasEnv(c, "LOKI_TOKEN")).To(BeFalse()) + }) + + It("passes loki.url and default queueDepth when Loki is configured", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki-gateway.monitoring.svc:3100/loki/api/v1/push" + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-loki-url=https://loki-gateway.monitoring.svc:3100/loki/api/v1/push")) + Expect(c.Args).To(ContainElement(jepDefaultLokiQueueDepthFlag)) + }) + + It("passes custom spec.telemetry.backpressure.queueDepth", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "http://loki:3100" + js.Spec.Telemetry.Backpressure.QueueDepth = 20000 + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-loki-queue-depth=20000")) + }) + + It("wires Loki credentials from spec.telemetry.loki.secretRef", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki:3100/loki/api/v1/push" + js.Spec.Telemetry.Loki.SecretRef = "loki-credentials" + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(envSecretRef(c, "LOKI_USERNAME")).To(Equal(secretKeyRef{"loki-credentials", "username"})) + Expect(envSecretRef(c, "LOKI_PASSWORD")).To(Equal(secretKeyRef{"loki-credentials", "password"})) + Expect(envSecretRef(c, "LOKI_TOKEN")).To(Equal(secretKeyRef{"loki-credentials", "token"})) + }) + + It("mounts loki.tls.caSecretRef and passes -loki-ca-file", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki:3100/loki/api/v1/push" + js.Spec.Telemetry.Loki.TLS.CASecretRef = "loki-ca-bundle" + + dep := createTelemetryDeployment(js, "") + c := dep.Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement(jepLokiCAFileFlag)) + + var mount *corev1.VolumeMount + for i := range c.VolumeMounts { + if c.VolumeMounts[i].Name == "loki-ca" { + mount = &c.VolumeMounts[i] + } + } + Expect(mount).NotTo(BeNil(), "expected loki-ca volume mount") + Expect(mount.MountPath).To(Equal("/loki-ca")) + Expect(mount.ReadOnly).To(BeTrue()) + + var vol *corev1.Volume + for i := range dep.Spec.Template.Spec.Volumes { + if dep.Spec.Template.Spec.Volumes[i].Name == "loki-ca" { + vol = &dep.Spec.Template.Spec.Volumes[i] + } + } + Expect(vol).NotTo(BeNil()) + Expect(vol.Secret).NotTo(BeNil()) + Expect(vol.Secret.SecretName).To(Equal("loki-ca-bundle")) + }) + + It("passes -loki-insecure-skip-verify when tls.insecureSkipVerify is set", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki:3100/loki/api/v1/push" + js.Spec.Telemetry.Loki.TLS.InsecureSkipVerify = true + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-loki-insecure-skip-verify=true")) + }) +}) + +type secretKeyRef struct { + name string + key string +} + +func envSecretRef(c corev1.Container, name string) secretKeyRef { + for _, e := range c.Env { + if e.Name == name && e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil { + return secretKeyRef{ + name: e.ValueFrom.SecretKeyRef.Name, + key: e.ValueFrom.SecretKeyRef.Key, + } + } + } + return secretKeyRef{} +} diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go new file mode 100644 index 000000000..76cbed5be --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/api/v1alpha1" +) + +// JEP-0013 operator defaults (Phase 3). Keep these literals in the tests so +// they encode the JEP rather than the implementation. +const ( + jepDefaultScrapeTimeoutFlag = "-scrape-timeout=7s" + jepDefaultDriverTypeEnumFlag = "-driver-type-enum=power,storage,network,serial,console,video,composite" + jepDefaultExemplarKeysFlag = "-exemplar-keys=client,lease_id" + jepMetricsBindFlag = "-metrics-bind-address=:8080" +) + +func phase3TelemetryJS(name, namespace string) *operatorv1alpha1.Jumpstarter { + return &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: false}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + } +} + +var _ = Describe("createTelemetryDeployment JEP-0013 Phase 3", func() { + It("passes default scrapeTimeout, driverTypeEnum, and exemplarKeys as flags", func() { + dep := createTelemetryDeployment(phase3TelemetryJS("js", "ns"), "") + c := dep.Spec.Template.Spec.Containers[0] + + Expect(c.Args).To(ContainElement(jepMetricsBindFlag)) + Expect(c.Args).To(ContainElement(jepDefaultScrapeTimeoutFlag)) + Expect(c.Args).To(ContainElement(jepDefaultDriverTypeEnumFlag)) + Expect(c.Args).To(ContainElement(jepDefaultExemplarKeysFlag)) + }) + + It("passes custom spec.telemetry.metrics fields as flags", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Metrics = operatorv1alpha1.TelemetryMetricsConfig{ + ExemplarKeys: []string{"client", "board-type"}, + DriverTypeEnum: []string{"power", "can"}, + ScrapeTimeout: &metav1.Duration{Duration: 3 * time.Second}, + } + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-scrape-timeout=3s")) + Expect(c.Args).To(ContainElement("-driver-type-enum=power,can")) + Expect(c.Args).To(ContainElement("-exemplar-keys=client,board-type")) + }) + + It("sets GRPC_TELEMETRY_ENDPOINT to the in-cluster telemetry Service", func() { + js := phase3TelemetryJS("js", "jumpstarter-lab") + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(envValue(c, "GRPC_TELEMETRY_ENDPOINT")).To(Equal(telemetryEndpointFor(js.Namespace))) + }) + + It("exposes container port metrics on 8080", func() { + c := createTelemetryDeployment(phase3TelemetryJS("js", "ns"), "").Spec.Template.Spec.Containers[0] + p := namedContainerPort(c.Ports, metricsPortName) + Expect(p).NotTo(BeNil(), "expected container port named metrics") + Expect(p.ContainerPort).To(Equal(int32(telemetryMetricsPort))) + Expect(p.Protocol).To(Equal(corev1.ProtocolTCP)) + }) +}) + +var _ = Describe("Telemetry ConfigMap certificate (JEP-0013 TLS)", func() { + const crName = "test-telemetry-ca" + + var crNamespace string + ctx := context.Background() + + BeforeEach(func() { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "telemetry-ca-"}} + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + crNamespace = ns.Name + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: crNamespace}, + }) + }) + + It("includes the CA PEM when cert-manager is enabled and the CA secret exists", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: true}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + }, + }, + } + Expect(k8sClient.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: crName + caCertificateSuffix, + Namespace: crNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte(testPEM)}, + })).To(Succeed()) + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + cfg, err := r.buildConfig(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Telemetry).NotTo(BeNil()) + Expect(cfg.Telemetry.Enabled).To(BeTrue()) + Expect(cfg.Telemetry.Endpoint).To(Equal(telemetryEndpointFor(crNamespace))) + Expect(cfg.Telemetry.Certificate).To(Equal(testPEM)) + }) + + It("omits the telemetry certificate when cert-manager is disabled", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: false}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + }, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + cfg, err := r.buildConfig(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Telemetry).NotTo(BeNil()) + Expect(cfg.Telemetry.Certificate).To(BeEmpty()) + }) +}) diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go index 77b2d2c25..978de4e4a 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go @@ -149,20 +149,27 @@ var _ = Describe("Telemetry Lifecycle", func() { Expect(deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot).NotTo(BeNil()) Expect(*deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot).To(BeTrue()) - By("verifying the Service exists") + By("verifying the Service exists with gRPC and metrics ports") svc := &corev1.Service{} Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: telemetryServiceName, Namespace: crNamespace, }, svc)).To(Succeed()) Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) - Expect(svc.Spec.Ports).To(HaveLen(1)) - Expect(svc.Spec.Ports[0].Port).To(Equal(int32(telemetryPort))) - Expect(svc.Spec.Ports[0].Name).To(Equal("grpc")) Expect(svc.Spec.Selector).To(HaveKeyWithValue("app", "jumpstarter-telemetry")) + + grpcP := namedServicePort(svc.Spec.Ports, grpcPortName) + Expect(grpcP).NotTo(BeNil(), "expected Service port named grpc") + Expect(grpcP.Port).To(Equal(int32(telemetryPort))) + Expect(grpcP.TargetPort.IntValue()).To(Equal(telemetryPort)) + + metricsP := namedServicePort(svc.Spec.Ports, metricsPortName) + Expect(metricsP).NotTo(BeNil(), "expected Service port named metrics (JEP-0013 /metrics scrape)") + Expect(metricsP.Port).To(Equal(int32(telemetryMetricsPort))) + Expect(metricsP.TargetPort.IntValue()).To(Equal(telemetryMetricsPort)) }) - It("includes liveness and readiness probes on the telemetry container", func() { + It("uses HTTP /healthz and /readyz on the metrics port (JEP-0013 DD-7)", func() { By("creating a Jumpstarter CR with telemetry enabled") spec := makeJumpstarterSpec() spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ @@ -184,15 +191,17 @@ var _ = Describe("Telemetry Lifecycle", func() { container := deployment.Spec.Template.Spec.Containers[0] - By("verifying liveness probe uses TCP on the gRPC port") + By("verifying liveness probe is HTTP GET /healthz on the metrics port") Expect(container.LivenessProbe).NotTo(BeNil()) - Expect(container.LivenessProbe.TCPSocket).NotTo(BeNil()) - Expect(container.LivenessProbe.TCPSocket.Port.IntValue()).To(Equal(telemetryPort)) + Expect(container.LivenessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/healthz")) + Expect(container.LivenessProbe.HTTPGet.Port.IntValue()).To(Equal(telemetryMetricsPort)) - By("verifying readiness probe uses TCP on the gRPC port") + By("verifying readiness probe is HTTP GET /readyz on the metrics port") Expect(container.ReadinessProbe).NotTo(BeNil()) - Expect(container.ReadinessProbe.TCPSocket).NotTo(BeNil()) - Expect(container.ReadinessProbe.TCPSocket.Port.IntValue()).To(Equal(telemetryPort)) + Expect(container.ReadinessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) + Expect(container.ReadinessProbe.HTTPGet.Port.IntValue()).To(Equal(telemetryMetricsPort)) }) It("respects the replicas field", func() { @@ -438,13 +447,14 @@ var _ = Describe("Telemetry Lifecycle", func() { container := dep.Spec.Template.Spec.Containers[0] - // Should have CONTROLLER_KEY + TLS env vars - Expect(container.Env).To(HaveLen(3)) + // CONTROLLER_KEY + GRPC_TELEMETRY_ENDPOINT + TLS env vars + Expect(container.Env).To(HaveLen(4)) envNames := make(map[string]string) for _, env := range container.Env { envNames[env.Name] = env.Value } Expect(envNames).To(HaveKey("CONTROLLER_KEY")) + Expect(envNames).To(HaveKey("GRPC_TELEMETRY_ENDPOINT")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_CERT_PEM", "/tls/tls.crt")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_KEY_PEM", "/tls/tls.key")) @@ -485,13 +495,14 @@ var _ = Describe("Telemetry Lifecycle", func() { container := dep.Spec.Template.Spec.Containers[0] - // Should have CONTROLLER_KEY + TLS env vars - Expect(container.Env).To(HaveLen(3)) + // CONTROLLER_KEY + GRPC_TELEMETRY_ENDPOINT + TLS env vars + Expect(container.Env).To(HaveLen(4)) envNames := make(map[string]string) for _, env := range container.Env { envNames[env.Name] = env.Value } Expect(envNames).To(HaveKey("CONTROLLER_KEY")) + Expect(envNames).To(HaveKey("GRPC_TELEMETRY_ENDPOINT")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_CERT_PEM", "/tls/tls.crt")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_KEY_PEM", "/tls/tls.key")) @@ -528,9 +539,15 @@ var _ = Describe("Telemetry Lifecycle", func() { container := dep.Spec.Template.Spec.Containers[0] - // Only CONTROLLER_KEY should be set (no TLS env vars) - Expect(container.Env).To(HaveLen(1)) - Expect(container.Env[0].Name).To(Equal("CONTROLLER_KEY")) + // CONTROLLER_KEY + GRPC_TELEMETRY_ENDPOINT (no TLS env vars) + Expect(container.Env).To(HaveLen(2)) + envNames := make(map[string]string) + for _, env := range container.Env { + envNames[env.Name] = env.Value + } + Expect(envNames).To(HaveKey("CONTROLLER_KEY")) + Expect(envNames).To(HaveKey("GRPC_TELEMETRY_ENDPOINT")) + Expect(envNames).NotTo(HaveKey("EXTERNAL_CERT_PEM")) // No volume mounts or volumes Expect(container.VolumeMounts).To(BeEmpty()) @@ -1066,3 +1083,39 @@ var _ = Describe("telemetryCANeedsRequeue", func() { Expect(r.telemetryCANeedsRequeue(ctx, js)).To(BeFalse()) }) }) + +func hasEnv(c corev1.Container, name string) bool { + for _, e := range c.Env { + if e.Name == name { + return true + } + } + return false +} + +func envValue(c corev1.Container, name string) string { + for _, e := range c.Env { + if e.Name == name { + return e.Value + } + } + return "" +} + +func namedContainerPort(ports []corev1.ContainerPort, name string) *corev1.ContainerPort { + for i := range ports { + if ports[i].Name == name { + return &ports[i] + } + } + return nil +} + +func namedServicePort(ports []corev1.ServicePort, name string) *corev1.ServicePort { + for i := range ports { + if ports[i].Name == name { + return &ports[i] + } + } + return nil +} diff --git a/controller/image_layout_test.go b/controller/image_layout_test.go new file mode 100644 index 000000000..0f6c541b2 --- /dev/null +++ b/controller/image_layout_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image_test + +import ( + "os" + "strings" + "testing" +) + +func mustRead(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func TestContainerfileBuildsTelemetryBinary(t *testing.T) { + content := mustRead(t, "Containerfile") + if !strings.Contains(content, "./cmd/telemetry") { + t.Error("Containerfile must compile ./cmd/telemetry") + } + if !strings.Contains(content, "-o telemetry") { + t.Error("Containerfile must emit a binary named telemetry") + } + if !strings.Contains(content, "COPY --from=builder /build/telemetry") { + t.Error("Containerfile must copy /build/telemetry into the runtime image as /telemetry") + } +} + +func TestMakefileBuildProducesTelemetry(t *testing.T) { + content := mustRead(t, "Makefile") + if !strings.Contains(content, "-o bin/telemetry") { + t.Error("make build must produce bin/telemetry") + } + if !strings.Contains(content, "cmd/telemetry") { + t.Error("make build must compile cmd/telemetry") + } +} + +func TestMakefileDockerBuildCIStagesTelemetry(t *testing.T) { + content := mustRead(t, "Makefile") + if !strings.Contains(content, "-o bin/ci-stage/controller/telemetry") { + t.Error("make docker-build-ci must stage telemetry next to manager and router so Containerfile.prebuilt COPY . . places /telemetry") + } +} diff --git a/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go b/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go index 00e583612..ee0ac7352 100644 --- a/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go +++ b/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Jumpstarter Authors +// Copyright 2026 The Jumpstarter Authors // Code generated by protoc-gen-go. DO NOT EDIT. // versions: @@ -24,6 +24,289 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Exporter → Telemetry +type MetricsStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Msg: + // + // *MetricsStreamRequest_Register + // *MetricsStreamRequest_ScrapeResponse + Msg isMetricsStreamRequest_Msg `protobuf_oneof:"msg"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsStreamRequest) Reset() { + *x = MetricsStreamRequest{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsStreamRequest) ProtoMessage() {} + +func (x *MetricsStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsStreamRequest.ProtoReflect.Descriptor instead. +func (*MetricsStreamRequest) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{0} +} + +func (x *MetricsStreamRequest) GetMsg() isMetricsStreamRequest_Msg { + if x != nil { + return x.Msg + } + return nil +} + +func (x *MetricsStreamRequest) GetRegister() *MetricsRegister { + if x != nil { + if x, ok := x.Msg.(*MetricsStreamRequest_Register); ok { + return x.Register + } + } + return nil +} + +func (x *MetricsStreamRequest) GetScrapeResponse() *MetricsScrapeResponse { + if x != nil { + if x, ok := x.Msg.(*MetricsStreamRequest_ScrapeResponse); ok { + return x.ScrapeResponse + } + } + return nil +} + +type isMetricsStreamRequest_Msg interface { + isMetricsStreamRequest_Msg() +} + +type MetricsStreamRequest_Register struct { + Register *MetricsRegister `protobuf:"bytes,1,opt,name=register,proto3,oneof"` // First message: identify this exporter. +} + +type MetricsStreamRequest_ScrapeResponse struct { + ScrapeResponse *MetricsScrapeResponse `protobuf:"bytes,2,opt,name=scrape_response,json=scrapeResponse,proto3,oneof"` // Subsequent: reply to a scrape. +} + +func (*MetricsStreamRequest_Register) isMetricsStreamRequest_Msg() {} + +func (*MetricsStreamRequest_ScrapeResponse) isMetricsStreamRequest_Msg() {} + +type MetricsRegister struct { + state protoimpl.MessageState `protogen:"open.v1"` + Identity string `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"` // Exporter CRD name (verified against the auth token by the server). + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsRegister) Reset() { + *x = MetricsRegister{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsRegister) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsRegister) ProtoMessage() {} + +func (x *MetricsRegister) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsRegister.ProtoReflect.Descriptor instead. +func (*MetricsRegister) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{1} +} + +func (x *MetricsRegister) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + +type MetricsScrapeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MetricsText []byte `protobuf:"bytes,1,opt,name=metrics_text,json=metricsText,proto3" json:"metrics_text,omitempty"` // generate_latest() OpenMetrics output. + Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsScrapeResponse) Reset() { + *x = MetricsScrapeResponse{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsScrapeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsScrapeResponse) ProtoMessage() {} + +func (x *MetricsScrapeResponse) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsScrapeResponse.ProtoReflect.Descriptor instead. +func (*MetricsScrapeResponse) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{2} +} + +func (x *MetricsScrapeResponse) GetMetricsText() []byte { + if x != nil { + return x.MetricsText + } + return nil +} + +func (x *MetricsScrapeResponse) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +// Telemetry → Exporter +type MetricsStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Msg: + // + // *MetricsStreamResponse_ScrapeRequest + Msg isMetricsStreamResponse_Msg `protobuf_oneof:"msg"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsStreamResponse) Reset() { + *x = MetricsStreamResponse{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsStreamResponse) ProtoMessage() {} + +func (x *MetricsStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsStreamResponse.ProtoReflect.Descriptor instead. +func (*MetricsStreamResponse) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{3} +} + +func (x *MetricsStreamResponse) GetMsg() isMetricsStreamResponse_Msg { + if x != nil { + return x.Msg + } + return nil +} + +func (x *MetricsStreamResponse) GetScrapeRequest() *MetricsScrapeRequest { + if x != nil { + if x, ok := x.Msg.(*MetricsStreamResponse_ScrapeRequest); ok { + return x.ScrapeRequest + } + } + return nil +} + +type isMetricsStreamResponse_Msg interface { + isMetricsStreamResponse_Msg() +} + +type MetricsStreamResponse_ScrapeRequest struct { + ScrapeRequest *MetricsScrapeRequest `protobuf:"bytes,1,opt,name=scrape_request,json=scrapeRequest,proto3,oneof"` +} + +func (*MetricsStreamResponse_ScrapeRequest) isMetricsStreamResponse_Msg() {} + +// Empty request: "send your /metrics now". +type MetricsScrapeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsScrapeRequest) Reset() { + *x = MetricsScrapeRequest{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsScrapeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsScrapeRequest) ProtoMessage() {} + +func (x *MetricsScrapeRequest) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsScrapeRequest.ProtoReflect.Descriptor instead. +func (*MetricsScrapeRequest) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{4} +} + // Request to push log entries to the telemetry service. type PushLogsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -34,7 +317,7 @@ type PushLogsRequest struct { func (x *PushLogsRequest) Reset() { *x = PushLogsRequest{} - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -46,7 +329,7 @@ func (x *PushLogsRequest) String() string { func (*PushLogsRequest) ProtoMessage() {} func (x *PushLogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -59,7 +342,7 @@ func (x *PushLogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogsRequest.ProtoReflect.Descriptor instead. func (*PushLogsRequest) Descriptor() ([]byte, []int) { - return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{0} + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{5} } func (x *PushLogsRequest) GetEntries() []*LogEntry { @@ -80,7 +363,7 @@ type PushLogsResponse struct { func (x *PushLogsResponse) Reset() { *x = PushLogsResponse{} - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -92,7 +375,7 @@ func (x *PushLogsResponse) String() string { func (*PushLogsResponse) ProtoMessage() {} func (x *PushLogsResponse) ProtoReflect() protoreflect.Message { - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -105,7 +388,7 @@ func (x *PushLogsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogsResponse.ProtoReflect.Descriptor instead. func (*PushLogsResponse) Descriptor() ([]byte, []int) { - return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{1} + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{6} } func (x *PushLogsResponse) GetAccepted() uint32 { @@ -125,26 +408,27 @@ func (x *PushLogsResponse) GetDropped() uint32 { // A structured log entry from an exporter or client. // Maps directly to a Loki log entry with stream labels and body fields. type LogEntry struct { - state protoimpl.MessageState `protogen:"open.v1"` - Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // When the log was emitted. - Severity string `protobuf:"bytes,2,opt,name=severity,proto3" json:"severity,omitempty"` // Log severity: debug, info, warning, error, critical. - Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"` // Human-readable log message. - Component string `protobuf:"bytes,4,opt,name=component,proto3" json:"component,omitempty"` // Loki stream label: cli, exporter, controller, router, telemetry. - Exporter string `protobuf:"bytes,5,opt,name=exporter,proto3" json:"exporter,omitempty"` // Loki stream label: exporter CRD name (bounded by cluster size). - Lease string `protobuf:"bytes,6,opt,name=lease,proto3" json:"lease,omitempty"` // Log body only (high cardinality): active lease name. - Client string `protobuf:"bytes,7,opt,name=client,proto3" json:"client,omitempty"` // Log body only (high cardinality): client CRD name. - Operation string `protobuf:"bytes,8,opt,name=operation,proto3" json:"operation,omitempty"` // Log body: operation name (flash, power, etc.). - Result string `protobuf:"bytes,9,opt,name=result,proto3" json:"result,omitempty"` // Log body: operation outcome (success, failure, etc.). - DriverType string `protobuf:"bytes,10,opt,name=driver_type,json=driverType,proto3" json:"driver_type,omitempty"` // Log body: driver category (storage, power, network, etc.). - ExtraFields map[string]string `protobuf:"bytes,11,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Additional structured key-value fields. - Namespace string `protobuf:"bytes,12,opt,name=namespace,proto3" json:"namespace,omitempty"` // Loki stream label: Kubernetes namespace (bounded by cluster size). + state protoimpl.MessageState `protogen:"open.v1"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // When the log was emitted. + Severity string `protobuf:"bytes,2,opt,name=severity,proto3" json:"severity,omitempty"` // Log severity: debug, info, warning, error, critical. + Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"` // Human-readable log message. + Component string `protobuf:"bytes,4,opt,name=component,proto3" json:"component,omitempty"` // Loki stream label: cli, exporter, controller, router, telemetry. + Exporter string `protobuf:"bytes,5,opt,name=exporter,proto3" json:"exporter,omitempty"` // Loki stream label: exporter CRD name (bounded by cluster size). + Lease string `protobuf:"bytes,6,opt,name=lease,proto3" json:"lease,omitempty"` // Log body only (high cardinality): active lease name. + Client string `protobuf:"bytes,7,opt,name=client,proto3" json:"client,omitempty"` // Log body only (high cardinality): client CRD name. + Operation string `protobuf:"bytes,8,opt,name=operation,proto3" json:"operation,omitempty"` // Log body: operation name (flash, power, etc.). + Result string `protobuf:"bytes,9,opt,name=result,proto3" json:"result,omitempty"` // Log body: operation outcome (success, failure, etc.). + DriverType string `protobuf:"bytes,10,opt,name=driver_type,json=driverType,proto3" json:"driver_type,omitempty"` // Log body: driver category (storage, power, network, etc.). + ExtraFields map[string]string `protobuf:"bytes,11,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Additional structured key-value fields. + // Capped at 16 entries, 64-char keys, 256-char values. + Namespace string `protobuf:"bytes,12,opt,name=namespace,proto3" json:"namespace,omitempty"` // Loki stream label: Kubernetes namespace (bounded by cluster size). unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *LogEntry) Reset() { *x = LogEntry{} - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -156,7 +440,7 @@ func (x *LogEntry) String() string { func (*LogEntry) ProtoMessage() {} func (x *LogEntry) ProtoReflect() protoreflect.Message { - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -169,7 +453,7 @@ func (x *LogEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. func (*LogEntry) Descriptor() ([]byte, []int) { - return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{2} + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{7} } func (x *LogEntry) GetTimestamp() *timestamppb.Timestamp { @@ -260,7 +544,20 @@ var File_jumpstarter_v1_telemetry_proto protoreflect.FileDescriptor const file_jumpstarter_v1_telemetry_proto_rawDesc = "" + "\n" + - "\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"E\n" + + "\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xae\x01\n" + + "\x14MetricsStreamRequest\x12=\n" + + "\bregister\x18\x01 \x01(\v2\x1f.jumpstarter.v1.MetricsRegisterH\x00R\bregister\x12P\n" + + "\x0fscrape_response\x18\x02 \x01(\v2%.jumpstarter.v1.MetricsScrapeResponseH\x00R\x0escrapeResponseB\x05\n" + + "\x03msg\"-\n" + + "\x0fMetricsRegister\x12\x1a\n" + + "\bidentity\x18\x01 \x01(\tR\bidentity\"t\n" + + "\x15MetricsScrapeResponse\x12!\n" + + "\fmetrics_text\x18\x01 \x01(\fR\vmetricsText\x128\n" + + "\ttimestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"m\n" + + "\x15MetricsStreamResponse\x12M\n" + + "\x0escrape_request\x18\x01 \x01(\v2$.jumpstarter.v1.MetricsScrapeRequestH\x00R\rscrapeRequestB\x05\n" + + "\x03msg\"\x16\n" + + "\x14MetricsScrapeRequest\"E\n" + "\x0fPushLogsRequest\x122\n" + "\aentries\x18\x01 \x03(\v2\x18.jumpstarter.v1.LogEntryR\aentries\"H\n" + "\x10PushLogsResponse\x12\x1a\n" + @@ -283,8 +580,9 @@ const file_jumpstarter_v1_telemetry_proto_rawDesc = "" + "\tnamespace\x18\f \x01(\tR\tnamespace\x1a>\n" + "\x10ExtraFieldsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x012a\n" + - "\x10TelemetryService\x12M\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x012\xc3\x01\n" + + "\x10TelemetryService\x12`\n" + + "\rMetricsStream\x12$.jumpstarter.v1.MetricsStreamRequest\x1a%.jumpstarter.v1.MetricsStreamResponse(\x010\x01\x12M\n" + "\bPushLogs\x12\x1f.jumpstarter.v1.PushLogsRequest\x1a .jumpstarter.v1.PushLogsResponseB\xdf\x01\n" + "\x12com.jumpstarter.v1B\x0eTelemetryProtoP\x01Z`github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1;jumpstarterv1\xa2\x02\x03JXX\xaa\x02\x0eJumpstarter.V1\xca\x02\x0eJumpstarter\\V1\xe2\x02\x1aJumpstarter\\V1\\GPBMetadata\xea\x02\x0fJumpstarter::V1b\x06proto3" @@ -300,25 +598,36 @@ func file_jumpstarter_v1_telemetry_proto_rawDescGZIP() []byte { return file_jumpstarter_v1_telemetry_proto_rawDescData } -var file_jumpstarter_v1_telemetry_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_jumpstarter_v1_telemetry_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_jumpstarter_v1_telemetry_proto_goTypes = []any{ - (*PushLogsRequest)(nil), // 0: jumpstarter.v1.PushLogsRequest - (*PushLogsResponse)(nil), // 1: jumpstarter.v1.PushLogsResponse - (*LogEntry)(nil), // 2: jumpstarter.v1.LogEntry - nil, // 3: jumpstarter.v1.LogEntry.ExtraFieldsEntry - (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp + (*MetricsStreamRequest)(nil), // 0: jumpstarter.v1.MetricsStreamRequest + (*MetricsRegister)(nil), // 1: jumpstarter.v1.MetricsRegister + (*MetricsScrapeResponse)(nil), // 2: jumpstarter.v1.MetricsScrapeResponse + (*MetricsStreamResponse)(nil), // 3: jumpstarter.v1.MetricsStreamResponse + (*MetricsScrapeRequest)(nil), // 4: jumpstarter.v1.MetricsScrapeRequest + (*PushLogsRequest)(nil), // 5: jumpstarter.v1.PushLogsRequest + (*PushLogsResponse)(nil), // 6: jumpstarter.v1.PushLogsResponse + (*LogEntry)(nil), // 7: jumpstarter.v1.LogEntry + nil, // 8: jumpstarter.v1.LogEntry.ExtraFieldsEntry + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp } var file_jumpstarter_v1_telemetry_proto_depIdxs = []int32{ - 2, // 0: jumpstarter.v1.PushLogsRequest.entries:type_name -> jumpstarter.v1.LogEntry - 4, // 1: jumpstarter.v1.LogEntry.timestamp:type_name -> google.protobuf.Timestamp - 3, // 2: jumpstarter.v1.LogEntry.extra_fields:type_name -> jumpstarter.v1.LogEntry.ExtraFieldsEntry - 0, // 3: jumpstarter.v1.TelemetryService.PushLogs:input_type -> jumpstarter.v1.PushLogsRequest - 1, // 4: jumpstarter.v1.TelemetryService.PushLogs:output_type -> jumpstarter.v1.PushLogsResponse - 4, // [4:5] is the sub-list for method output_type - 3, // [3:4] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 1, // 0: jumpstarter.v1.MetricsStreamRequest.register:type_name -> jumpstarter.v1.MetricsRegister + 2, // 1: jumpstarter.v1.MetricsStreamRequest.scrape_response:type_name -> jumpstarter.v1.MetricsScrapeResponse + 9, // 2: jumpstarter.v1.MetricsScrapeResponse.timestamp:type_name -> google.protobuf.Timestamp + 4, // 3: jumpstarter.v1.MetricsStreamResponse.scrape_request:type_name -> jumpstarter.v1.MetricsScrapeRequest + 7, // 4: jumpstarter.v1.PushLogsRequest.entries:type_name -> jumpstarter.v1.LogEntry + 9, // 5: jumpstarter.v1.LogEntry.timestamp:type_name -> google.protobuf.Timestamp + 8, // 6: jumpstarter.v1.LogEntry.extra_fields:type_name -> jumpstarter.v1.LogEntry.ExtraFieldsEntry + 0, // 7: jumpstarter.v1.TelemetryService.MetricsStream:input_type -> jumpstarter.v1.MetricsStreamRequest + 5, // 8: jumpstarter.v1.TelemetryService.PushLogs:input_type -> jumpstarter.v1.PushLogsRequest + 3, // 9: jumpstarter.v1.TelemetryService.MetricsStream:output_type -> jumpstarter.v1.MetricsStreamResponse + 6, // 10: jumpstarter.v1.TelemetryService.PushLogs:output_type -> jumpstarter.v1.PushLogsResponse + 9, // [9:11] is the sub-list for method output_type + 7, // [7:9] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_jumpstarter_v1_telemetry_proto_init() } @@ -326,13 +635,20 @@ func file_jumpstarter_v1_telemetry_proto_init() { if File_jumpstarter_v1_telemetry_proto != nil { return } + file_jumpstarter_v1_telemetry_proto_msgTypes[0].OneofWrappers = []any{ + (*MetricsStreamRequest_Register)(nil), + (*MetricsStreamRequest_ScrapeResponse)(nil), + } + file_jumpstarter_v1_telemetry_proto_msgTypes[3].OneofWrappers = []any{ + (*MetricsStreamResponse_ScrapeRequest)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_jumpstarter_v1_telemetry_proto_rawDesc), len(file_jumpstarter_v1_telemetry_proto_rawDesc)), NumEnums: 0, - NumMessages: 4, + NumMessages: 9, NumExtensions: 0, NumServices: 1, }, diff --git a/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go b/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go index 356b656a2..d5db4a9a7 100644 --- a/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go +++ b/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Jumpstarter Authors +// Copyright 2026 The Jumpstarter Authors // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: @@ -21,16 +21,20 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - TelemetryService_PushLogs_FullMethodName = "/jumpstarter.v1.TelemetryService/PushLogs" + TelemetryService_MetricsStream_FullMethodName = "/jumpstarter.v1.TelemetryService/MetricsStream" + TelemetryService_PushLogs_FullMethodName = "/jumpstarter.v1.TelemetryService/PushLogs" ) // TelemetryServiceClient is the client API for TelemetryService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// A service that receives structured logs from exporters and clients. +// A service that reverse-scrapes exporter metrics and receives structured logs. // Implemented by jumpstarter-telemetry; not part of the controller. type TelemetryServiceClient interface { + // Persistent bidirectional stream: telemetry sends scrape requests, + // exporter responds with full metric snapshots (OpenMetrics text). + MetricsStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[MetricsStreamRequest, MetricsStreamResponse], error) // Push structured log entries to the telemetry service for Loki ingest. PushLogs(ctx context.Context, in *PushLogsRequest, opts ...grpc.CallOption) (*PushLogsResponse, error) } @@ -43,6 +47,19 @@ func NewTelemetryServiceClient(cc grpc.ClientConnInterface) TelemetryServiceClie return &telemetryServiceClient{cc} } +func (c *telemetryServiceClient) MetricsStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[MetricsStreamRequest, MetricsStreamResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TelemetryService_ServiceDesc.Streams[0], TelemetryService_MetricsStream_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[MetricsStreamRequest, MetricsStreamResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TelemetryService_MetricsStreamClient = grpc.BidiStreamingClient[MetricsStreamRequest, MetricsStreamResponse] + func (c *telemetryServiceClient) PushLogs(ctx context.Context, in *PushLogsRequest, opts ...grpc.CallOption) (*PushLogsResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PushLogsResponse) @@ -57,9 +74,12 @@ func (c *telemetryServiceClient) PushLogs(ctx context.Context, in *PushLogsReque // All implementations must embed UnimplementedTelemetryServiceServer // for forward compatibility. // -// A service that receives structured logs from exporters and clients. +// A service that reverse-scrapes exporter metrics and receives structured logs. // Implemented by jumpstarter-telemetry; not part of the controller. type TelemetryServiceServer interface { + // Persistent bidirectional stream: telemetry sends scrape requests, + // exporter responds with full metric snapshots (OpenMetrics text). + MetricsStream(grpc.BidiStreamingServer[MetricsStreamRequest, MetricsStreamResponse]) error // Push structured log entries to the telemetry service for Loki ingest. PushLogs(context.Context, *PushLogsRequest) (*PushLogsResponse, error) mustEmbedUnimplementedTelemetryServiceServer() @@ -72,6 +92,9 @@ type TelemetryServiceServer interface { // pointer dereference when methods are called. type UnimplementedTelemetryServiceServer struct{} +func (UnimplementedTelemetryServiceServer) MetricsStream(grpc.BidiStreamingServer[MetricsStreamRequest, MetricsStreamResponse]) error { + return status.Error(codes.Unimplemented, "method MetricsStream not implemented") +} func (UnimplementedTelemetryServiceServer) PushLogs(context.Context, *PushLogsRequest) (*PushLogsResponse, error) { return nil, status.Error(codes.Unimplemented, "method PushLogs not implemented") } @@ -96,6 +119,13 @@ func RegisterTelemetryServiceServer(s grpc.ServiceRegistrar, srv TelemetryServic s.RegisterService(&TelemetryService_ServiceDesc, srv) } +func _TelemetryService_MetricsStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TelemetryServiceServer).MetricsStream(&grpc.GenericServerStream[MetricsStreamRequest, MetricsStreamResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TelemetryService_MetricsStreamServer = grpc.BidiStreamingServer[MetricsStreamRequest, MetricsStreamResponse] + func _TelemetryService_PushLogs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PushLogsRequest) if err := dec(in); err != nil { @@ -126,6 +156,13 @@ var TelemetryService_ServiceDesc = grpc.ServiceDesc{ Handler: _TelemetryService_PushLogs_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "MetricsStream", + Handler: _TelemetryService_MetricsStream_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, Metadata: "jumpstarter/v1/telemetry.proto", } diff --git a/controller/internal/service/loki_push.go b/controller/internal/service/loki_push.go new file mode 100644 index 000000000..e1c929725 --- /dev/null +++ b/controller/internal/service/loki_push.go @@ -0,0 +1,425 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultLokiQueueDepth = 10000 + droppedTotalMetric = "jumpstarter_telemetry_dropped_total" + droppedDestination = "loki" + lokiPushPath = "/loki/api/v1/push" + lokiFlushInterval = 500 * time.Millisecond + lokiHTTPTimeout = 10 * time.Second +) + +// LokiConfig is the telemetry-side Loki HTTP push configuration. +type LokiConfig struct { + URL string + Username string + Password string + Token string + CAFile string + InsecureSkipVerify bool + QueueDepth int +} + +type lokiPushPayload struct { + Streams []lokiStream `json:"streams"` +} + +type lokiStream struct { + Stream map[string]string `json:"stream"` + Values [][]string `json:"values"` +} + +// LokiPusher is a bounded ring buffer that POSTs LogEntry batches to Loki. +type LokiPusher struct { + url string + username string + password string + token string + client *http.Client + queueDepth int + dropped prometheus.Counter + now func() time.Time + + mu sync.Mutex + entries []*pb.LogEntry + dropCount int + dropFirst time.Time +} + +func normalizeQueueDepth(d int) int { + if d <= 0 { + return defaultLokiQueueDepth + } + if d < 2 { + return 2 + } + return d +} + +func normalizeLokiPushURL(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("loki url: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("loki url: unsupported scheme %q (use http or https)", u.Scheme) + } + if u.Path == "" || u.Path == "/" { + u.Path = lokiPushPath + } + return u.String(), nil +} + +func lokiHTTPClient(cfg LokiConfig) (*http.Client, error) { + transport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return &http.Client{Timeout: lokiHTTPTimeout}, nil + } + transport = transport.Clone() + tlsCfg := &tls.Config{MinVersion: tls.VersionTLS12} + customTLS := false + if cfg.InsecureSkipVerify { + tlsCfg.InsecureSkipVerify = true + customTLS = true + } + if cfg.CAFile != "" { + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return nil, fmt.Errorf("loki ca file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("loki ca file: no certificates found in %s", cfg.CAFile) + } + tlsCfg.RootCAs = pool + customTLS = true + } + if customTLS { + transport.TLSClientConfig = tlsCfg + } + return &http.Client{Timeout: lokiHTTPTimeout, Transport: transport}, nil +} + +// NewLokiPusher returns nil, nil when URL is empty (metrics-only). grpc:// is rejected. +func NewLokiPusher(cfg LokiConfig, dropped *prometheus.CounterVec) (*LokiPusher, error) { + raw := strings.TrimSpace(cfg.URL) + if raw == "" { + return nil, nil + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("loki url: %w", err) + } + if u.Scheme == "grpc" || u.Scheme == "grpcs" { + return nil, fmt.Errorf("loki gRPC scheme %q is not implemented; use http:// or https://", u.Scheme) + } + pushURL, err := normalizeLokiPushURL(raw) + if err != nil { + return nil, err + } + client, err := lokiHTTPClient(cfg) + if err != nil { + return nil, err + } + var counter prometheus.Counter + if dropped != nil { + counter = dropped.WithLabelValues(droppedDestination) + } + return &LokiPusher{ + url: pushURL, + username: cfg.Username, + password: cfg.Password, + token: cfg.Token, + client: client, + queueDepth: normalizeQueueDepth(cfg.QueueDepth), + dropped: counter, + now: time.Now, + }, nil +} + +func (p *LokiPusher) maxReal() int { + n := p.queueDepth - 1 + if n < 1 { + return 1 + } + return n +} + +// Enqueue is non-blocking. On overflow, entries are replaced by a single drop marker. +func (p *LokiPusher) Enqueue(entry *pb.LogEntry) { + if p == nil || entry == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if len(p.entries) < p.maxReal() { + p.entries = append(p.entries, entry) + return + } + if p.dropCount == 0 { + p.dropFirst = p.now() + } + p.dropCount++ + if p.dropped != nil { + p.dropped.Inc() + } +} + +func makeDropMarker(count int, first, now time.Time) *pb.LogEntry { + window := 0 + if !first.IsZero() { + window = int(now.Sub(first).Seconds()) + } + return &pb.LogEntry{ + Timestamp: timestamppb.New(now), + Severity: "warning", + Message: "log entries dropped due to Loki backpressure", + Component: "telemetry", + Operation: "backpressure", + ExtraFields: map[string]string{ + "count": strconv.Itoa(count), + "window_seconds": strconv.Itoa(window), + }, + } +} + +func (p *LokiPusher) dropMarkerLocked() *pb.LogEntry { + return makeDropMarker(p.dropCount, p.dropFirst, p.now()) +} + +func (p *LokiPusher) queued() []*pb.LogEntry { + p.mu.Lock() + defer p.mu.Unlock() + out := append([]*pb.LogEntry(nil), p.entries...) + if p.dropCount > 0 { + out = append(out, p.dropMarkerLocked()) + } + return out +} + +func (p *LokiPusher) takeBatch() (entries []*pb.LogEntry, dropCount int, dropFirst time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + entries = p.entries + p.entries = nil + dropCount = p.dropCount + dropFirst = p.dropFirst + p.dropCount = 0 + p.dropFirst = time.Time{} + return entries, dropCount, dropFirst +} + +func (p *LokiPusher) restoreBatch(entries []*pb.LogEntry, dropCount int, dropFirst time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + p.entries = append(entries, p.entries...) + if dropCount > 0 { + if p.dropCount == 0 || (!dropFirst.IsZero() && (p.dropFirst.IsZero() || dropFirst.Before(p.dropFirst))) { + p.dropFirst = dropFirst + } + p.dropCount += dropCount + } +} + +// Flush POSTs queued entries to Loki. On HTTP failure the buffer is restored. +func (p *LokiPusher) Flush(ctx context.Context) error { + if p == nil { + return nil + } + entries, dropCount, dropFirst := p.takeBatch() + batch := append([]*pb.LogEntry(nil), entries...) + if dropCount > 0 { + batch = append(batch, makeDropMarker(dropCount, dropFirst, p.now())) + } + if len(batch) == 0 { + return nil + } + if err := p.push(ctx, batch); err != nil { + p.restoreBatch(entries, dropCount, dropFirst) + return err + } + return nil +} + +func (p *LokiPusher) push(ctx context.Context, entries []*pb.LogEntry) error { + body, err := json.Marshal(buildLokiPayload(entries, p.now())) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if p.token != "" { + req.Header.Set("Authorization", "Bearer "+p.token) + } else if p.username != "" || p.password != "" { + req.SetBasicAuth(p.username, p.password) + } + resp, err := p.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + slurp, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return fmt.Errorf("loki push %s: %s", resp.Status, bytes.TrimSpace(slurp)) + } + return nil +} + +// Run flushes the buffer periodically until ctx is cancelled, then flushes once more. +func (p *LokiPusher) Run(ctx context.Context) { + if p == nil { + return + } + ticker := time.NewTicker(lokiFlushInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + flushCtx, cancel := context.WithTimeout(context.Background(), lokiHTTPTimeout) + _ = p.Flush(flushCtx) + cancel() + return + case <-ticker.C: + _ = p.Flush(ctx) + } + } +} + +func buildLokiPayload(entries []*pb.LogEntry, now time.Time) lokiPushPayload { + type acc struct { + labels map[string]string + values [][]string + } + order := make([]string, 0) + grouped := make(map[string]*acc) + for _, e := range entries { + labels := lokiStreamLabels(e) + key := labels["component"] + "\x00" + labels["exporter"] + "\x00" + labels["namespace"] + g, ok := grouped[key] + if !ok { + g = &acc{labels: labels} + grouped[key] = g + order = append(order, key) + } + g.values = append(g.values, []string{unixNanoString(e, now), logLineJSON(e)}) + } + streams := make([]lokiStream, 0, len(order)) + for _, key := range order { + g := grouped[key] + streams = append(streams, lokiStream{Stream: g.labels, Values: g.values}) + } + return lokiPushPayload{Streams: streams} +} + +func lokiStreamLabels(e *pb.LogEntry) map[string]string { + m := make(map[string]string, 3) + if e.Component != "" { + m["component"] = e.Component + } + if e.Exporter != "" { + m["exporter"] = e.Exporter + } + if e.Namespace != "" { + m["namespace"] = e.Namespace + } + return m +} + +func unixNanoString(e *pb.LogEntry, now time.Time) string { + ts := now + if e.Timestamp != nil && e.Timestamp.IsValid() { + ts = e.Timestamp.AsTime() + } + return strconv.FormatInt(ts.UnixNano(), 10) +} + +func logLineJSON(e *pb.LogEntry) string { + body := map[string]string{} + if e.Message != "" { + body["msg"] = e.Message + } + if e.Severity != "" { + body["severity"] = e.Severity + } + if e.Component != "" { + body["component"] = e.Component + } + if e.Exporter != "" { + body["exporter"] = e.Exporter + } + if e.Namespace != "" { + body["namespace"] = e.Namespace + } + if e.Lease != "" { + body["lease"] = e.Lease + } + if e.Client != "" { + body["client"] = e.Client + } + if e.Operation != "" { + body["operation"] = e.Operation + } + if e.Result != "" { + body["result"] = e.Result + } + if e.DriverType != "" { + body["driver_type"] = e.DriverType + } + if e.Timestamp != nil && e.Timestamp.IsValid() { + body["ts"] = e.Timestamp.AsTime().Format(time.RFC3339Nano) + } + for k, v := range e.ExtraFields { + if _, reserved := reservedExtraFieldKeys[k]; reserved { + continue + } + if k == "" { + continue + } + body[k] = v + } + b, err := json.Marshal(body) + if err != nil { + return `{"msg":"marshal error"}` + } + return string(b) +} diff --git a/controller/internal/service/loki_push_test.go b/controller/internal/service/loki_push_test.go new file mode 100644 index 000000000..966407fd6 --- /dev/null +++ b/controller/internal/service/loki_push_test.go @@ -0,0 +1,400 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func testDroppedCounter() *prometheus.CounterVec { + return prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: droppedTotalMetric, + Help: "Log entries dropped due to Loki backpressure.", + }, []string{"destination"}) +} + +func logEntry(msg string) *pb.LogEntry { + return &pb.LogEntry{ + Timestamp: timestamppb.New(time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)), + Severity: "info", + Message: msg, + Component: "exporter", + Exporter: "lab-1", + Namespace: "jumpstarter", + Lease: "lease-1", + Client: "ci", + Operation: "flash", + Result: "success", + } +} + +func TestNewLokiPusher_EmptyURLIsNil(t *testing.T) { + p, err := NewLokiPusher(LokiConfig{}, testDroppedCounter()) + if err != nil { + t.Fatalf("empty URL: %v", err) + } + if p != nil { + t.Fatal("metrics-only deployments must not construct a Loki pusher") + } +} + +func TestNewLokiPusher_GRPCSchemeNotImplemented(t *testing.T) { + p, err := NewLokiPusher(LokiConfig{URL: "grpc://loki.monitoring.svc:9095"}, testDroppedCounter()) + if p != nil { + t.Fatal("gRPC Loki must not start an HTTP pusher") + } + if err == nil { + t.Fatal("expected error for unimplemented grpc:// Loki URL") + } +} + +func TestNormalizeLokiPushURL_AppendsPath(t *testing.T) { + got, err := normalizeLokiPushURL("http://loki:3100") + if err != nil { + t.Fatal(err) + } + if got != "http://loki:3100/loki/api/v1/push" { + t.Errorf("got %q", got) + } +} + +func TestNormalizeLokiPushURL_KeepsExistingPath(t *testing.T) { + raw := "https://loki-gateway.monitoring.svc:3100/loki/api/v1/push" + got, err := normalizeLokiPushURL(raw) + if err != nil { + t.Fatal(err) + } + if got != raw { + t.Errorf("got %q", got) + } +} + +func TestDefaultQueueDepthIs10000(t *testing.T) { + if d := normalizeQueueDepth(0); d != defaultLokiQueueDepth { + t.Errorf("normalizeQueueDepth(0) = %d, want %d", d, defaultLokiQueueDepth) + } + if defaultLokiQueueDepth != 10000 { + t.Errorf("default Loki queue depth = %d, want 10000 (JEP-0013)", defaultLokiQueueDepth) + } +} + +func TestLokiBuffer_OverflowEmitsDropMarker(t *testing.T) { + dropped := testDroppedCounter() + p, err := NewLokiPusher(LokiConfig{URL: "http://127.0.0.1:1", QueueDepth: 3}, dropped) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + p.now = func() time.Time { return now } + + p.Enqueue(logEntry("a")) + p.Enqueue(logEntry("b")) + p.Enqueue(logEntry("c")) // overflow: reserved marker slot + now = now.Add(12 * time.Second) + p.Enqueue(logEntry("d")) // accumulate into the same marker + + queued := p.queued() + if len(queued) != 3 { + t.Fatalf("queued = %d, want 2 real entries + 1 drop marker", len(queued)) + } + if queued[0].Message != "a" || queued[1].Message != "b" { + t.Fatalf("real entries = %q, %q; want a, b", queued[0].Message, queued[1].Message) + } + marker := queued[2] + if marker.Severity != "warning" { + t.Errorf("marker severity = %q, want warning", marker.Severity) + } + if marker.Component != "telemetry" { + t.Errorf("marker component = %q, want telemetry", marker.Component) + } + if marker.Operation != "backpressure" { + t.Errorf("marker operation = %q, want backpressure", marker.Operation) + } + if marker.ExtraFields["count"] != "2" { + t.Errorf("marker count = %q, want 2", marker.ExtraFields["count"]) + } + if marker.ExtraFields["window_seconds"] != "12" { + t.Errorf("marker window_seconds = %q, want 12", marker.ExtraFields["window_seconds"]) + } + + got := counterValue(t, dropped, "loki") + if got != 2 { + t.Errorf("dropped_total{destination=loki} = %v, want 2", got) + } +} + +func TestLokiPush_HTTPPostsOpenMetricsCompatibleJSON(t *testing.T) { + var ( + mu sync.Mutex + gotPath string + gotAuth string + gotCT string + gotBody []byte + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + gotCT = r.Header.Get("Content-Type") + gotBody = body + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + dropped := testDroppedCounter() + p, err := NewLokiPusher(LokiConfig{ + URL: srv.URL, + Username: "loki", + Password: "secret", + QueueDepth: 10, + }, dropped) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("hello loki")) + if err := p.Flush(context.Background()); err != nil { + t.Fatalf("Flush: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if gotPath != "/loki/api/v1/push" { + t.Errorf("path = %q, want /loki/api/v1/push", gotPath) + } + if !strings.HasPrefix(gotCT, "application/json") { + t.Errorf("Content-Type = %q, want application/json", gotCT) + } + if !strings.HasPrefix(gotAuth, "Basic ") { + t.Errorf("Authorization = %q, want Basic", gotAuth) + } + + var payload lokiPushPayload + if err := json.Unmarshal(gotBody, &payload); err != nil { + t.Fatalf("unmarshal %s: %v", gotBody, err) + } + if len(payload.Streams) != 1 { + t.Fatalf("streams = %d, want 1", len(payload.Streams)) + } + stream := payload.Streams[0].Stream + if stream["component"] != "exporter" || stream["exporter"] != "lab-1" || stream["namespace"] != "jumpstarter" { + t.Errorf("stream labels = %v, want component/exporter/namespace", stream) + } + for _, highCard := range []string{"client", "lease", "lease_id", "operation"} { + if _, ok := stream[highCard]; ok { + t.Errorf("high-cardinality field %q must not be a Loki stream label", highCard) + } + } + if len(payload.Streams[0].Values) != 1 { + t.Fatalf("values = %d, want 1", len(payload.Streams[0].Values)) + } + ts, line := payload.Streams[0].Values[0][0], payload.Streams[0].Values[0][1] + if _, err := strconv.ParseInt(ts, 10, 64); err != nil { + t.Errorf("timestamp %q is not unix nanoseconds", ts) + } + var body map[string]string + if err := json.Unmarshal([]byte(line), &body); err != nil { + t.Fatalf("log line JSON: %v", err) + } + if body["msg"] != "hello loki" { + t.Errorf("msg = %q", body["msg"]) + } + if body["severity"] != "info" { + t.Errorf("severity = %q", body["severity"]) + } + if body["client"] != "ci" || body["lease"] != "lease-1" { + t.Errorf("high-cardinality fields missing from JSON body: %v", body) + } +} + +func TestLokiPush_FlushFailureKeepsBuffer(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + + p, err := NewLokiPusher(LokiConfig{URL: srv.URL, QueueDepth: 10}, testDroppedCounter()) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("keep me")) + if err := p.Flush(context.Background()); err == nil { + t.Fatal("expected flush error") + } + if len(p.queued()) != 1 { + t.Fatalf("failed flush must keep the buffer, queued = %d", len(p.queued())) + } +} + +func TestLokiPush_BearerToken(t *testing.T) { + var gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + p, err := NewLokiPusher(LokiConfig{URL: srv.URL, Token: "s3cret", QueueDepth: 8}, testDroppedCounter()) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("tok")) + if err := p.Flush(context.Background()); err != nil { + t.Fatal(err) + } + if gotAuth != "Bearer s3cret" { + t.Errorf("Authorization = %q, want Bearer s3cret", gotAuth) + } +} + +func TestDroppedTotalRegisteredOnMetrics(t *testing.T) { + svc := &TelemetryService{} + svc.initScrapeTimeouts() + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatal(err) + } + var found bool + var dest string + for _, mf := range mfs { + if mf.GetName() != droppedTotalMetric { + continue + } + found = true + if len(mf.Metric) > 0 { + for _, lp := range mf.Metric[0].GetLabel() { + if lp.GetName() == "destination" { + dest = lp.GetValue() + } + } + } + } + if !found { + t.Fatalf("%s missing from /metrics registry", droppedTotalMetric) + } + if dest != "loki" { + t.Errorf("destination label = %q, want loki", dest) + } +} + +func TestPushLogs_ForwardsAcceptedEntriesToLoki(t *testing.T) { + var gotBody []byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + signer := testSigner(t) + dropped := testDroppedCounter() + pusher, err := NewLokiPusher(LokiConfig{URL: srv.URL, QueueDepth: 10}, dropped) + if err != nil { + t.Fatal(err) + } + svc := &TelemetryService{BindAddr: ":0", Signer: signer, Loki: pusher} + ctx := authedCtx(t, signer, "exporter:jumpstarter:real-exporter:uid1") + + resp, err := svc.PushLogs(ctx, &pb.PushLogsRequest{Entries: []*pb.LogEntry{{ + Severity: "info", + Message: "from exporter", + Component: "exporter", + Exporter: "spoofed", + Namespace: "jumpstarter", + }}}) + if err != nil { + t.Fatal(err) + } + if resp.Dropped != 1 { + t.Fatalf("Dropped = %d, want 1 (identity mismatch is not Loki backpressure)", resp.Dropped) + } + if n := len(pusher.queued()); n != 0 { + t.Fatalf("identity-mismatch entries must not be queued for Loki, queued=%d", n) + } + + resp, err = svc.PushLogs(ctx, &pb.PushLogsRequest{Entries: []*pb.LogEntry{{ + Severity: "info", + Message: "ok", + Component: "exporter", + Exporter: "real-exporter", + Namespace: "jumpstarter", + Client: "ci", + }}}) + if err != nil { + t.Fatal(err) + } + if resp.Accepted != 1 { + t.Fatalf("Accepted = %d, want 1", resp.Accepted) + } + if err := pusher.Flush(context.Background()); err != nil { + t.Fatal(err) + } + var payload lokiPushPayload + if err := json.Unmarshal(gotBody, &payload); err != nil { + t.Fatalf("body %s: %v", gotBody, err) + } + if payload.Streams[0].Stream["exporter"] != "real-exporter" { + t.Errorf("stream exporter = %q (must come from token, not the payload)", payload.Streams[0].Stream["exporter"]) + } +} + +func counterValue(t *testing.T, vec *prometheus.CounterVec, dest string) float64 { + t.Helper() + m, err := vec.GetMetricWithLabelValues(dest) + if err != nil { + t.Fatal(err) + } + var pb dto.Metric + if err := m.Write(&pb); err != nil { + t.Fatal(err) + } + return pb.GetCounter().GetValue() +} + +func TestLokiPush_InsecureSkipVerify(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + p, err := NewLokiPusher(LokiConfig{ + URL: srv.URL, + InsecureSkipVerify: true, + QueueDepth: 4, + }, testDroppedCounter()) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("tls")) + if err := p.Flush(context.Background()); err != nil { + t.Fatalf("Flush with insecureSkipVerify: %v", err) + } +} diff --git a/controller/internal/service/metrics_merge.go b/controller/internal/service/metrics_merge.go new file mode 100644 index 000000000..484b4e27e --- /dev/null +++ b/controller/internal/service/metrics_merge.go @@ -0,0 +1,246 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "errors" + "io" + "sort" + "strings" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +const ( + labelExporter = "exporter" + labelDriverType = "driver_type" + driverTypeOther = "other" + + // scrapeTimeoutsMetric is incremented once per exporter that does not + // answer a MetricsStream scrape within scrapeTimeout (JEP-0013). + scrapeTimeoutsMetric = "jumpstarter_scrape_timeouts_total" + + // metricsParseErrorsMetric is incremented once per exporter snapshot that + // MetricsStream delivered but OpenMetrics parse rejected (JEP-0013). + metricsParseErrorsMetric = "jumpstarter_metrics_parse_errors_total" +) + +// DefaultDriverTypeEnum is the JEP-0013 default allowlist for driver_type. +var DefaultDriverTypeEnum = []string{ + "power", "storage", "network", "serial", "console", "video", "composite", +} + +// DefaultExemplarKeys is the JEP-0013 default exemplar allowlist. +var DefaultExemplarKeys = []string{"client", "lease_id"} + +type mergeConfig struct { + exporterName string + driverTypes map[string]struct{} + exemplarKeys map[string]struct{} +} + +type exporterSnapshot struct { + name string + text []byte +} + +func setToMap(values []string) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, v := range values { + if v == "" { + continue + } + out[v] = struct{}{} + } + return out +} + +func (s *TelemetryService) mergeConfigFor(name string) mergeConfig { + types := s.DriverTypeEnum + if len(types) == 0 { + types = DefaultDriverTypeEnum + } + keys := s.ExemplarKeys + if len(keys) == 0 { + keys = DefaultExemplarKeys + } + return mergeConfig{ + exporterName: name, + driverTypes: setToMap(types), + exemplarKeys: setToMap(keys), + } +} + +func parseMetricFamilies(text []byte) ([]*dto.MetricFamily, error) { + dec := expfmt.NewDecoder(bytes.NewReader(text), expfmt.NewFormat(expfmt.TypeOpenMetrics)) + var out []*dto.MetricFamily + for { + mf := new(dto.MetricFamily) + err := dec.Decode(mf) + if errors.Is(err, io.EOF) { + return out, nil + } + if err != nil { + return nil, err + } + out = append(out, mf) + } +} + +func encodeMetricFamilies(w io.Writer, families []*dto.MetricFamily) error { + sorted := append([]*dto.MetricFamily(nil), families...) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].GetName() < sorted[j].GetName() + }) + enc := expfmt.NewEncoder(w, expfmt.NewFormat(expfmt.TypeOpenMetrics)) + for _, f := range sorted { + if f == nil { + continue + } + if err := enc.Encode(f); err != nil { + return err + } + } + if closer, ok := enc.(expfmt.Closer); ok { + return closer.Close() + } + return nil +} + +func mergeSnapshots(snapshots []exporterSnapshot, extra []*dto.MetricFamily, cfgFor func(string) mergeConfig, onParseError func(exporter string, err error)) []*dto.MetricFamily { + byName := map[string]*dto.MetricFamily{} + for _, f := range extra { + if f == nil || f.GetName() == "" { + continue + } + byName[f.GetName()] = f + } + for _, snap := range snapshots { + if len(snap.text) == 0 { + continue + } + families, err := parseMetricFamilies(snap.text) + if err != nil { + if onParseError != nil { + onParseError(snap.name, err) + } + continue + } + cfg := cfgFor(snap.name) + for _, f := range families { + applyFamily(f, cfg) + existing, ok := byName[f.GetName()] + if !ok { + byName[f.GetName()] = f + continue + } + existing.Metric = append(existing.Metric, f.Metric...) + } + } + out := make([]*dto.MetricFamily, 0, len(byName)) + for _, f := range byName { + out = append(out, f) + } + return out +} + +func applyFamily(f *dto.MetricFamily, cfg mergeConfig) { + for _, m := range f.GetMetric() { + applyMetric(m, cfg) + } +} + +func applyMetric(m *dto.Metric, cfg mergeConfig) { + setLabel(m, labelExporter, cfg.exporterName) + if v, ok := getLabel(m, labelDriverType); ok { + setLabel(m, labelDriverType, remapDriverType(v, cfg.driverTypes)) + } + if m.Counter != nil { + m.Counter.Exemplar = filterExemplar(m.Counter.Exemplar, cfg.exemplarKeys) + } + if m.Histogram != nil { + for _, b := range m.Histogram.Bucket { + if b != nil { + b.Exemplar = filterExemplar(b.Exemplar, cfg.exemplarKeys) + } + } + if len(m.Histogram.Exemplars) > 0 { + filtered := make([]*dto.Exemplar, 0, len(m.Histogram.Exemplars)) + for _, ex := range m.Histogram.Exemplars { + if fe := filterExemplar(ex, cfg.exemplarKeys); fe != nil { + filtered = append(filtered, fe) + } + } + m.Histogram.Exemplars = filtered + } + } +} + +func remapDriverType(value string, allow map[string]struct{}) string { + if _, ok := allow[value]; ok { + return value + } + return driverTypeOther +} + +func getLabel(m *dto.Metric, name string) (string, bool) { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + return lp.GetValue(), true + } + } + return "", false +} + +func setLabel(m *dto.Metric, name, value string) { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + v := value + lp.Value = &v + return + } + } + m.Label = append(m.Label, labelPair(name, value)) +} + +func labelPair(name, value string) *dto.LabelPair { + n, v := name, value + return &dto.LabelPair{Name: &n, Value: &v} +} + +func filterExemplar(ex *dto.Exemplar, allow map[string]struct{}) *dto.Exemplar { + if ex == nil { + return nil + } + kept := make([]*dto.LabelPair, 0, len(ex.GetLabel())) + for _, lp := range ex.GetLabel() { + if _, ok := allow[lp.GetName()]; !ok { + continue + } + if strings.TrimSpace(lp.GetValue()) == "" { + continue + } + kept = append(kept, lp) + } + if len(kept) == 0 { + return nil + } + ex.Label = kept + return ex +} diff --git a/controller/internal/service/metrics_merge_test.go b/controller/internal/service/metrics_merge_test.go new file mode 100644 index 000000000..72bdbbbc4 --- /dev/null +++ b/controller/internal/service/metrics_merge_test.go @@ -0,0 +1,262 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "errors" + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +func labeledCounterValue(t *testing.T, mfs []*dto.MetricFamily, name, label, want string) float64 { + t.Helper() + for _, mf := range mfs { + if mf.GetName() != name && mf.GetName()+"_total" != name { + continue + } + for _, m := range mf.Metric { + for _, lp := range m.GetLabel() { + if lp.GetName() == label && lp.GetValue() == want { + return m.GetCounter().GetValue() + } + } + } + } + t.Fatalf("%s{%s=%q} missing from gathered families", name, label, want) + return 0 +} + +func TestApplyMetric_OverwritesExporterRemapsDriverTypeAndFiltersExemplars(t *testing.T) { + one := 1.0 + m := &dto.Metric{ + Label: []*dto.LabelPair{ + labelPair("exporter", "spoofed"), + labelPair("driver_type", "tuya"), + labelPair("operation", "on"), + }, + Counter: &dto.Counter{ + Value: &one, + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + labelPair("client", "ci"), + labelPair("lease_id", "abc"), + labelPair("trace_id", "drop-me"), + }, + }, + }, + } + applyMetric(m, mergeConfig{ + exporterName: "sidekick", + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + }) + + if got, _ := getLabel(m, "exporter"); got != "sidekick" { + t.Errorf("exporter label = %q, want sidekick", got) + } + if got, _ := getLabel(m, "driver_type"); got != "other" { + t.Errorf("driver_type = %q, want other", got) + } + ex := m.Counter.GetExemplar() + if ex == nil { + t.Fatal("expected exemplar to be kept") + } + got := map[string]string{} + for _, lp := range ex.GetLabel() { + got[lp.GetName()] = lp.GetValue() + } + if got["client"] != "ci" || got["lease_id"] != "abc" { + t.Errorf("exemplar labels = %v, want client=ci lease_id=abc", got) + } + if _, ok := got["trace_id"]; ok { + t.Errorf("trace_id should have been dropped, got %v", got) + } +} + +func TestApplyMetric_KeepsAllowlistedDriverType(t *testing.T) { + m := &dto.Metric{ + Label: []*dto.LabelPair{ + labelPair("driver_type", "power"), + }, + } + applyMetric(m, mergeConfig{ + exporterName: "exp-a", + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + }) + if got, _ := getLabel(m, "driver_type"); got != "power" { + t.Errorf("driver_type = %q, want power", got) + } + if got, _ := getLabel(m, "exporter"); got != "exp-a" { + t.Errorf("exporter = %q, want exp-a", got) + } +} + +// pythonOpenMetricsWithExemplar is the prometheus_client OpenMetrics form +// that prometheus/common v0.62's text fallback cannot parse (it treats '#' as +// a timestamp). This is the body exporters send after driver operations. +const pythonOpenMetricsWithExemplar = `# TYPE jumpstarter_operation_duration_seconds histogram +jumpstarter_operation_duration_seconds_bucket{exporter="sidekick",le="0.005",operation="on",result="success",driver_type="power"} 1.0 # {lease_id="lease-1"} 0.00262 1788299670.149 +jumpstarter_operation_duration_seconds_sum{exporter="sidekick",operation="on",result="success",driver_type="power"} 0.00262 +jumpstarter_operation_duration_seconds_count{exporter="sidekick",operation="on",result="success",driver_type="power"} 1.0 +# EOF +` + +func testMergeCfg(name string) mergeConfig { + return mergeConfig{ + exporterName: name, + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + } +} + +func TestParseMetricFamilies_OpenMetricsExemplarSuffixFails(t *testing.T) { + _, err := parseMetricFamilies([]byte(pythonOpenMetricsWithExemplar)) + if err == nil { + t.Fatal("expected parse error for OpenMetrics exemplar suffix") + } +} + +func TestMergeSnapshots_CombinesExportersAndReportsInvalidText(t *testing.T) { + textA := []byte(`# TYPE jumpstarter_operations_total counter +# HELP jumpstarter_operations_total Total operations performed. +jumpstarter_operations_total{exporter="a",operation="on",result="success",driver_type="power"} 2.0 +# EOF +`) + textB := []byte(`# TYPE jumpstarter_operations_total counter +jumpstarter_operations_total{exporter="b",operation="off",result="success",driver_type="power"} 3.0 +# EOF +`) + type parseErrCall struct { + exporter string + err error + } + var calls []parseErrCall + families := mergeSnapshots([]exporterSnapshot{ + {name: "exp-a", text: textA}, + {name: "exp-b", text: textB}, + {name: "bad", text: []byte("not metrics")}, + }, nil, testMergeCfg, func(exporter string, err error) { + calls = append(calls, parseErrCall{exporter: exporter, err: err}) + }) + + var ops *dto.MetricFamily + for _, f := range families { + if f.GetName() == "jumpstarter_operations_total" { + ops = f + break + } + } + if ops == nil { + t.Fatal("missing jumpstarter_operations_total") + } + if len(ops.Metric) != 2 { + t.Fatalf("got %d series, want 2 (invalid snapshot omitted)", len(ops.Metric)) + } + exporters := map[string]bool{} + for _, m := range ops.Metric { + name, _ := getLabel(m, "exporter") + exporters[name] = true + } + if !exporters["exp-a"] || !exporters["exp-b"] { + t.Errorf("exporters = %v, want exp-a and exp-b", exporters) + } + if len(calls) != 1 { + t.Fatalf("parse-error callbacks = %d, want 1", len(calls)) + } + if calls[0].exporter != "bad" { + t.Errorf("parse-error exporter = %q, want bad", calls[0].exporter) + } + if calls[0].err == nil { + t.Fatal("parse-error callback err is nil") + } +} + +func TestMergeSnapshots_ReportsOpenMetricsExemplarParseError(t *testing.T) { + var gotExporter string + var gotErr error + families := mergeSnapshots([]exporterSnapshot{ + {name: "sidekick", text: []byte(pythonOpenMetricsWithExemplar)}, + }, nil, testMergeCfg, func(exporter string, err error) { + gotExporter = exporter + gotErr = err + }) + for _, f := range families { + if strings.HasPrefix(f.GetName(), "jumpstarter_operation_duration_seconds") { + t.Fatalf("unparseable snapshot must be omitted, got family %s", f.GetName()) + } + } + if gotExporter != "sidekick" { + t.Errorf("parse-error exporter = %q, want sidekick", gotExporter) + } + if gotErr == nil { + t.Fatal("parse-error callback err is nil") + } +} + +func TestRecordMetricsParseError_IncrementsLabeledCounter(t *testing.T) { + svc := &TelemetryService{} + svc.initScrapeTimeouts() + svc.recordMetricsParseError("sidekick", errors.New("parse failed")) + + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + value := labeledCounterValue(t, mfs, metricsParseErrorsMetric, labelExporter, "sidekick") + if value != 1 { + t.Fatalf("%s{exporter=sidekick} = %v, want 1", metricsParseErrorsMetric, value) + } +} + +func TestEncodeMetricFamilies_WritesOpenMetricsEOF(t *testing.T) { + name := "jumpstarter_scrape_timeouts_total" + help := "Exporter MetricsStream scrapes that exceeded scrapeTimeout." + metricType := dto.MetricType_COUNTER + zero := 0.0 + families := []*dto.MetricFamily{{ + Name: &name, + Help: &help, + Type: &metricType, + Metric: []*dto.Metric{{ + Counter: &dto.Counter{Value: &zero}, + }}, + }} + var buf bytes.Buffer + if err := encodeMetricFamilies(&buf, families); err != nil { + t.Fatalf("encode: %v", err) + } + body := buf.String() + if !strings.Contains(body, scrapeTimeoutsMetric) { + t.Errorf("encoded body missing %s:\n%s", scrapeTimeoutsMetric, body) + } + if !strings.Contains(body, "# EOF") { + t.Errorf("OpenMetrics body missing # EOF:\n%s", body) + } + dec := expfmt.NewDecoder(strings.NewReader(body), expfmt.NewFormat(expfmt.TypeOpenMetrics)) + mf := new(dto.MetricFamily) + if err := dec.Decode(mf); err != nil { + t.Fatalf("re-decode: %v", err) + } + if mf.GetName() != scrapeTimeoutsMetric { + t.Errorf("decoded name = %q", mf.GetName()) + } +} diff --git a/controller/internal/service/metrics_stream.go b/controller/internal/service/metrics_stream.go new file mode 100644 index 000000000..b16bb78ef --- /dev/null +++ b/controller/internal/service/metrics_stream.go @@ -0,0 +1,241 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "errors" + "sync" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +var errScrapeTimeout = errors.New("metrics scrape timeout") + +const defaultScrapeTimeout = 7 * time.Second + +// metricsConn is one authenticated MetricsStream. Scrapes are serialized +// because MetricsScrapeRequest has no scrape id (JEP-0013). +type metricsConn struct { + id exporterIdentity + scrapeMu sync.Mutex + mu sync.Mutex + send func(*pb.MetricsStreamResponse) error + // pending is non-nil only while a scrape is waiting for a response. + pending chan *pb.MetricsScrapeResponse + done <-chan struct{} + // started is closed when the Recv loop is running. + started chan struct{} +} + +func (s *TelemetryService) scrapeTimeoutDuration() time.Duration { + if s.ScrapeTimeout > 0 { + return s.ScrapeTimeout + } + return defaultScrapeTimeout +} + +func (s *TelemetryService) initMetricsState() { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.conns == nil { + s.conns = make(map[string]*metricsConn) + } +} + +func (s *TelemetryService) registerConn(c *metricsConn) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.conns == nil { + s.conns = make(map[string]*metricsConn) + } + s.conns[c.id.key()] = c +} + +func (s *TelemetryService) unregisterConn(c *metricsConn) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.conns == nil { + return + } + if cur, ok := s.conns[c.id.key()]; ok && cur == c { + delete(s.conns, c.id.key()) + } +} + +func (s *TelemetryService) snapshotConns() []*metricsConn { + s.stateMu.Lock() + defer s.stateMu.Unlock() + out := make([]*metricsConn, 0, len(s.conns)) + for _, c := range s.conns { + out = append(out, c) + } + return out +} + +// MetricsStream reverse-scrapes one exporter. The first message must be +// MetricsRegister whose identity matches the bearer token. +func (s *TelemetryService) MetricsStream(stream pb.TelemetryService_MetricsStreamServer) error { + s.initMetricsState() + id, err := s.authenticateExporter(stream.Context()) + if err != nil { + return err + } + + first, err := stream.Recv() + if err != nil { + return err + } + reg := first.GetRegister() + if reg == nil { + return status.Error(codes.InvalidArgument, "first MetricsStream message must be MetricsRegister") + } + if reg.GetIdentity() != id.name { + return status.Errorf(codes.PermissionDenied, "register identity %q does not match authenticated exporter %q", reg.GetIdentity(), id.name) + } + + done := make(chan struct{}) + conn := &metricsConn{ + id: id, + send: stream.Send, + done: done, + started: make(chan struct{}), + } + s.registerConn(conn) + defer s.unregisterConn(conn) + + logger := log.FromContext(stream.Context()).WithName("telemetry").WithValues( + "exporter", id.name, + "namespace", id.namespace, + ) + logger.Info("MetricsStream registered") + + var closeDone sync.Once + finish := func() { closeDone.Do(func() { close(done) }) } + defer finish() + close(conn.started) + + for { + msg, err := stream.Recv() + if err != nil { + return err + } + resp := msg.GetScrapeResponse() + if resp == nil { + return status.Error(codes.InvalidArgument, "expected MetricsScrapeResponse after register") + } + conn.mu.Lock() + pending := conn.pending + conn.pending = nil + conn.mu.Unlock() + if pending == nil { + continue + } + select { + case pending <- resp: + case <-done: + return nil + case <-stream.Context().Done(): + return stream.Context().Err() + } + } +} + +func (c *metricsConn) scrape(ctx context.Context, timeout time.Duration) ([]byte, error) { + c.scrapeMu.Lock() + defer c.scrapeMu.Unlock() + + reply := make(chan *pb.MetricsScrapeResponse, 1) + c.mu.Lock() + c.pending = reply + c.mu.Unlock() + defer func() { + c.mu.Lock() + if c.pending == reply { + c.pending = nil + } + c.mu.Unlock() + }() + + err := c.send(&pb.MetricsStreamResponse{ + Msg: &pb.MetricsStreamResponse_ScrapeRequest{ + ScrapeRequest: &pb.MetricsScrapeRequest{}, + }, + }) + if err != nil { + return nil, err + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case resp := <-reply: + if resp == nil { + return nil, errScrapeTimeout + } + return resp.GetMetricsText(), nil + case <-timer.C: + return nil, errScrapeTimeout + case <-ctx.Done(): + return nil, ctx.Err() + case <-c.done: + return nil, context.Canceled + } +} + +func (s *TelemetryService) fanoutScrapes(ctx context.Context) []exporterSnapshot { + s.initMetricsState() + s.initScrapeTimeouts() + timeout := s.scrapeTimeoutDuration() + conns := s.snapshotConns() + if len(conns) == 0 { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var mu sync.Mutex + var snaps []exporterSnapshot + var wg sync.WaitGroup + for _, c := range conns { + wg.Add(1) + go func(c *metricsConn) { + defer wg.Done() + text, err := c.scrape(ctx, timeout) + if err != nil { + if errors.Is(err, errScrapeTimeout) || errors.Is(err, context.DeadlineExceeded) { + s.scrapeTimeouts.Inc() + } + log.FromContext(ctx).WithName("telemetry").V(1).Info("exporter scrape omitted", + "exporter", c.id.name, + "error", err.Error(), + ) + return + } + mu.Lock() + snaps = append(snaps, exporterSnapshot{name: c.id.name, text: text}) + mu.Unlock() + }(c) + } + wg.Wait() + return snaps +} diff --git a/controller/internal/service/metrics_stream_test.go b/controller/internal/service/metrics_stream_test.go new file mode 100644 index 000000000..6aed15c36 --- /dev/null +++ b/controller/internal/service/metrics_stream_test.go @@ -0,0 +1,469 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" +) + +const bufconnSize = 1024 * 1024 + +func startTestHub(t *testing.T, timeout time.Duration) (*TelemetryService, pb.TelemetryServiceClient, string) { + t.Helper() + signer := testSigner(t) + svc := &TelemetryService{ + Signer: signer, + MetricsBindAddr: "127.0.0.1:0", + ScrapeTimeout: timeout, + DriverTypeEnum: DefaultDriverTypeEnum, + ExemplarKeys: DefaultExemplarKeys, + } + svc.grpcReady.Store(true) + + shutdown, err := svc.startMetricsHTTP() + if err != nil { + t.Fatalf("startMetricsHTTP: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = shutdown(ctx) + }) + + lis := bufconn.Listen(bufconnSize) + gs := grpc.NewServer() + pb.RegisterTelemetryServiceServer(gs, svc) + go func() { _ = gs.Serve(lis) }() + t.Cleanup(func() { + gs.Stop() + _ = lis.Close() + }) + + conn, err := grpc.NewClient("passthrough:///bufnet", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return lis.DialContext(ctx) + }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Fatalf("grpc.NewClient: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + + return svc, pb.NewTelemetryServiceClient(conn), svc.metricsAddr +} + +func exporterStreamCtx(t *testing.T, svc *TelemetryService, subject string) context.Context { + t.Helper() + token, err := svc.Signer.Token(subject) + if err != nil { + t.Fatalf("Token: %v", err) + } + return metadata.NewOutgoingContext( + context.Background(), + metadata.Pairs("authorization", "Bearer "+token), + ) +} + +func waitRegistered(t *testing.T, svc *TelemetryService, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + conns := svc.snapshotConns() + if len(conns) != n { + time.Sleep(10 * time.Millisecond) + continue + } + ready := true + for _, c := range conns { + if c.started == nil { + ready = false + break + } + select { + case <-c.started: + default: + ready = false + } + if !ready { + break + } + } + if ready { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %d MetricsStream connections, have %d", n, len(svc.snapshotConns())) +} + +func httpGet(t *testing.T, url string) (int, string) { + t.Helper() + client := &http.Client{Timeout: 3 * time.Second} + var resp *http.Response + var lastErr error + for i := 0; i < 30; i++ { + resp, lastErr = client.Get(url) + if lastErr == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if lastErr != nil { + t.Fatalf("GET %s: %v", url, lastErr) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + return resp.StatusCode, string(body) +} + +func TestHealthzAndReadyz(t *testing.T) { + _, _, addr := startTestHub(t, 200*time.Millisecond) + code, body := httpGet(t, "http://"+addr+"/healthz") + if code != http.StatusOK || !strings.Contains(body, "ok") { + t.Fatalf("healthz status=%d body=%q", code, body) + } + code, body = httpGet(t, "http://"+addr+"/readyz") + if code != http.StatusOK || !strings.Contains(body, "ok") { + t.Fatalf("readyz status=%d body=%q", code, body) + } +} + +func TestReadyzNotReadyWhenGRPCDown(t *testing.T) { + svc := &TelemetryService{MetricsBindAddr: "127.0.0.1:0"} + shutdown, err := svc.startMetricsHTTP() + if err != nil { + t.Fatalf("startMetricsHTTP: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = shutdown(ctx) + }) + code, _ := httpGet(t, "http://"+svc.metricsAddr+"/readyz") + if code != http.StatusServiceUnavailable { + t.Fatalf("readyz status=%d, want 503", code) + } +} + +func TestMetricsStream_RejectsIdentityMismatch(t *testing.T) { + svc, client, _ := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:alice:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "bob"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + _, err = stream.Recv() + if err == nil { + t.Fatal("expected PermissionDenied, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("code = %v, want PermissionDenied (%v)", status.Code(err), err) + } +} + +func TestMetricsStream_RejectsNonRegisterFirstMessage(t *testing.T) { + svc, client, _ := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:alice:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_ScrapeResponse{ + ScrapeResponse: &pb.MetricsScrapeResponse{MetricsText: []byte("# EOF\n")}, + }, + }); err != nil { + t.Fatalf("Send: %v", err) + } + _, err = stream.Recv() + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("code = %v, want InvalidArgument (%v)", status.Code(err), err) + } +} + +func TestMetricsStream_RejectsNonExporterToken(t *testing.T) { + svc, client, _ := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "client:jumpstarter:ci:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "ci"}, + }, + }); err != nil { + t.Fatalf("Send: %v", err) + } + _, err = stream.Recv() + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("code = %v, want PermissionDenied (%v)", status.Code(err), err) + } +} + +func TestMetricsConn_ScrapeTimeout(t *testing.T) { + done := make(chan struct{}) + c := &metricsConn{ + send: func(*pb.MetricsStreamResponse) error { return nil }, + done: done, + } + _, err := c.scrape(context.Background(), 40*time.Millisecond) + if !errors.Is(err, errScrapeTimeout) { + t.Fatalf("err = %v, want errScrapeTimeout", err) + } +} + +func TestFanout_TimeoutIncrementsCounterWithoutGRPC(t *testing.T) { + svc := &TelemetryService{ScrapeTimeout: 40 * time.Millisecond} + svc.initScrapeTimeouts() + done := make(chan struct{}) + c := &metricsConn{ + id: exporterIdentity{namespace: "jumpstarter", name: "slow"}, + send: func(*pb.MetricsStreamResponse) error { return nil }, + done: done, + } + svc.registerConn(c) + snaps := svc.fanoutScrapes(context.Background()) + if len(snaps) != 0 { + t.Fatalf("timed-out scrape must be omitted, got %d snapshots", len(snaps)) + } + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + var value float64 + var found bool + for _, mf := range mfs { + if mf.GetName() == scrapeTimeoutsMetric || mf.GetName()+"_total" == scrapeTimeoutsMetric { + found = true + if len(mf.Metric) > 0 { + value = mf.Metric[0].GetCounter().GetValue() + } + } + } + if !found || value < 1 { + t.Fatalf("scrape timeout counter = %v found=%v, want >= 1", value, found) + } +} + +func TestFanout_TimeoutOmitsExporterAndIncrementsCounter(t *testing.T) { + svc, client, addr := startTestHub(t, 150*time.Millisecond) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:slow:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "slow"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + // Drain scrape requests so Send on the server does not block, but never reply. + go func() { + for { + if _, err := stream.Recv(); err != nil { + return + } + } + }() + waitRegistered(t, svc, 1) + + code, body := httpGet(t, "http://"+addr+"/metrics") + if code != http.StatusOK { + t.Fatalf("GET /metrics status=%d body=%s", code, body) + } + if strings.Contains(body, "jumpstarter_operations_total") { + t.Errorf("timed-out exporter metrics must be omitted, body:\n%s", body) + } + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + var value float64 + var found bool + for _, mf := range mfs { + if mf.GetName() == scrapeTimeoutsMetric || mf.GetName()+"_total" == scrapeTimeoutsMetric { + found = true + if len(mf.Metric) > 0 { + value = mf.Metric[0].GetCounter().GetValue() + } + } + } + if !found || value < 1 { + t.Fatalf("scrape timeout counter = %v found=%v body:\n%s", value, found, body) + } +} + +func TestFanout_MergesOpenMetricsFromConnectedExporter(t *testing.T) { + svc, client, addr := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:sidekick:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "sidekick"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + + snapshot := []byte(`# TYPE jumpstarter_operations_total counter +# HELP jumpstarter_operations_total Total operations performed. +jumpstarter_operations_total{exporter="spoofed",operation="on",result="success",driver_type="tuya"} 4.0 +# EOF +`) + errCh := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + errCh <- err + return + } + if msg.GetScrapeRequest() == nil { + continue + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_ScrapeResponse{ + ScrapeResponse: &pb.MetricsScrapeResponse{MetricsText: snapshot}, + }, + }); err != nil { + errCh <- err + return + } + } + }() + waitRegistered(t, svc, 1) + + code, body := httpGet(t, "http://"+addr+"/metrics") + if code != http.StatusOK { + t.Fatalf("GET /metrics status=%d body=%s", code, body) + } + if !strings.Contains(body, `exporter="sidekick"`) { + t.Errorf("authenticated exporter label missing:\n%s", body) + } + if strings.Contains(body, `exporter="spoofed"`) { + t.Errorf("spoofed exporter label must be overwritten:\n%s", body) + } + if !strings.Contains(body, `driver_type="other"`) { + t.Errorf("unknown driver_type must remap to other:\n%s", body) + } + select { + case err := <-errCh: + if err != nil && err != io.EOF && status.Code(err) != codes.Canceled && status.Code(err) != codes.Unavailable { + t.Fatalf("stream goroutine: %v", err) + } + default: + } +} + +func TestFanout_UnparseableSnapshotIncrementsParseErrorsOnSameResponse(t *testing.T) { + svc, client, addr := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:sidekick:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "sidekick"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + + errCh := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + errCh <- err + return + } + if msg.GetScrapeRequest() == nil { + continue + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_ScrapeResponse{ + ScrapeResponse: &pb.MetricsScrapeResponse{MetricsText: []byte(pythonOpenMetricsWithExemplar)}, + }, + }); err != nil { + errCh <- err + return + } + } + }() + waitRegistered(t, svc, 1) + + code, body := httpGet(t, "http://"+addr+"/metrics") + if code != http.StatusOK { + t.Fatalf("GET /metrics status=%d body=%s", code, body) + } + if strings.Contains(body, "jumpstarter_operation_duration_seconds") { + t.Errorf("unparseable exporter snapshot must be omitted, body:\n%s", body) + } + if !strings.Contains(body, metricsParseErrorsMetric) || !strings.Contains(body, `exporter="sidekick"`) { + t.Errorf("parse-error counter missing from same /metrics response:\n%s", body) + } + + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + value := labeledCounterValue(t, mfs, metricsParseErrorsMetric, labelExporter, "sidekick") + if value < 1 { + t.Fatalf("%s{exporter=sidekick} = %v, want >= 1 body:\n%s", metricsParseErrorsMetric, value, body) + } + + select { + case err := <-errCh: + if err != nil && err != io.EOF && status.Code(err) != codes.Canceled && status.Code(err) != codes.Unavailable { + t.Fatalf("stream goroutine: %v", err) + } + default: + } +} diff --git a/controller/internal/service/telemetry_http.go b/controller/internal/service/telemetry_http.go new file mode 100644 index 000000000..91357cf34 --- /dev/null +++ b/controller/internal/service/telemetry_http.go @@ -0,0 +1,148 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" + ctrl "sigs.k8s.io/controller-runtime" +) + +func (s *TelemetryService) initScrapeTimeouts() { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.metricsRegistry != nil { + return + } + s.metricsRegistry = prometheus.NewRegistry() + s.scrapeTimeouts = prometheus.NewCounter(prometheus.CounterOpts{ + Name: scrapeTimeoutsMetric, + Help: "Exporter MetricsStream scrapes that exceeded scrapeTimeout.", + }) + s.parseErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: metricsParseErrorsMetric, + Help: "Exporter MetricsStream snapshots omitted because OpenMetrics parse failed.", + }, []string{labelExporter}) + s.droppedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: droppedTotalMetric, + Help: "Log entries dropped due to Loki backpressure.", + }, []string{"destination"}) + s.metricsRegistry.MustRegister(s.scrapeTimeouts, s.parseErrors, s.droppedTotal) + s.droppedTotal.WithLabelValues(droppedDestination).Add(0) +} + +func (s *TelemetryService) recordMetricsParseError(exporter string, err error) { + if s.parseErrors != nil { + s.parseErrors.WithLabelValues(exporter).Inc() + } + ctrl.Log.WithName("telemetry").Error(err, "exporter metrics snapshot omitted", logFieldExporter, exporter) +} + +func metricsHTTPEnabled(addr string) bool { + return addr != "" && addr != "0" +} + +func (s *TelemetryService) startMetricsHTTP() (func(context.Context) error, error) { + addr := s.MetricsBindAddr + if !metricsHTTPEnabled(addr) { + return nil, nil + } + s.initMetricsState() + s.initScrapeTimeouts() + + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + s.metricsAddr = ln.Addr().String() + + mux := http.NewServeMux() + mux.HandleFunc("/metrics", s.handleMetrics) + mux.HandleFunc("/healthz", s.handleHealthz) + mux.HandleFunc("/readyz", s.handleReadyz) + + writeTimeout := s.scrapeTimeoutDuration() + 15*time.Second + if writeTimeout < 30*time.Second { + writeTimeout = 30 * time.Second + } + + srv := &http.Server{ + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: writeTimeout, + IdleTimeout: 5 * time.Minute, + } + go func() { + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { + ctrl.Log.WithName("telemetry").Error(err, "metrics HTTP server stopped unexpectedly") + } + }() + ctrl.Log.WithName("telemetry").Info("Telemetry metrics HTTP listening", + "addr", s.metricsAddr, + ) + return srv.Shutdown, nil +} + +func (s *TelemetryService) handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + +// handleReadyz is 200 once the gRPC listener is bound. +// Loki is optional (metrics-only is valid) and is not a readiness gate. +func (s *TelemetryService) handleReadyz(w http.ResponseWriter, _ *http.Request) { + if !s.grpcReady.Load() { + http.Error(w, "grpc not ready\n", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + +func (s *TelemetryService) handleMetrics(w http.ResponseWriter, r *http.Request) { + s.initScrapeTimeouts() + snaps := s.fanoutScrapes(r.Context()) + // Merge exporter snapshots first so parse failures increment parseErrors + // before hub metrics are gathered into the same /metrics response. + families := mergeSnapshots(snaps, nil, s.mergeConfigFor, s.recordMetricsParseError) + + if s.metricsRegistry != nil { + gathered, err := s.metricsRegistry.Gather() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + families = mergeSnapshots(nil, append(gathered, families...), s.mergeConfigFor, nil) + } + + var buf bytes.Buffer + if err := encodeMetricFamilies(&buf, families); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", string(expfmt.NewFormat(expfmt.TypeOpenMetrics))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(buf.Bytes()) +} diff --git a/controller/internal/service/telemetry_identity.go b/controller/internal/service/telemetry_identity.go new file mode 100644 index 000000000..e1f23de4e --- /dev/null +++ b/controller/internal/service/telemetry_identity.go @@ -0,0 +1,62 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "strings" + + "github.com/jumpstarter-dev/jumpstarter/controller/internal/authentication" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// exporterIdentity is the exporter CRD namespace/name claimed by a bearer token. +type exporterIdentity struct { + namespace string + name string +} + +func (id exporterIdentity) key() string { + return id.namespace + "/" + id.name +} + +func parseExporterSubject(subject string) (exporterIdentity, error) { + parts := strings.SplitN(subject, ":", 4) + if len(parts) != 4 || parts[0] != "exporter" { + return exporterIdentity{}, status.Errorf(codes.PermissionDenied, "token is not an exporter token") + } + if parts[1] == "" || parts[2] == "" { + return exporterIdentity{}, status.Errorf(codes.PermissionDenied, "token has incomplete exporter identity") + } + return exporterIdentity{namespace: parts[1], name: parts[2]}, nil +} + +func (s *TelemetryService) authenticateExporter(ctx context.Context) (exporterIdentity, error) { + token, err := authentication.BearerTokenFromContext(ctx) + if err != nil { + return exporterIdentity{}, err + } + if s.Signer == nil { + return exporterIdentity{}, status.Error(codes.Internal, "telemetry signer is not configured") + } + subject, err := s.Signer.ParseSubject(token) + if err != nil { + return exporterIdentity{}, status.Errorf(codes.Unauthenticated, "invalid token: %v", err) + } + return parseExporterSubject(subject) +} diff --git a/controller/internal/service/telemetry_service.go b/controller/internal/service/telemetry_service.go index 80844b807..05ecafaeb 100644 --- a/controller/internal/service/telemetry_service.go +++ b/controller/internal/service/telemetry_service.go @@ -22,17 +22,17 @@ import ( "fmt" "net" "strings" + "sync" + "sync/atomic" "time" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" - "github.com/jumpstarter-dev/jumpstarter/controller/internal/authentication" "github.com/jumpstarter-dev/jumpstarter/controller/internal/oidc" pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" - "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/reflection" - "google.golang.org/grpc/status" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -59,7 +59,7 @@ var reservedExtraFieldKeys = map[string]struct{}{ } // TelemetryService receives structured log entries from exporters and clients, -// logs them via structured stdout, and will forward them to Loki in a future phase. +// logs them via structured stdout, and optionally forwards them to Loki. // // TLS: the server always uses TLS. When EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM // env vars point to certificate/key files (mounted by the operator from a Secret), @@ -69,42 +69,55 @@ var reservedExtraFieldKeys = map[string]struct{}{ type TelemetryService struct { pb.UnimplementedTelemetryServiceServer - // BindAddr is the TCP address to listen on (e.g. ":9093"). + // BindAddr is the TCP address to listen on for gRPC (e.g. ":9093"). BindAddr string - // Signer is used to validate bearer tokens on every PushLogs call. - // Tokens are issued by the controller from the same CONTROLLER_KEY seed, - // so the telemetry binary can verify them locally without a k8s client. + // MetricsBindAddr is the TCP address for HTTP GET /metrics, /healthz, and + // /readyz. Empty or "0" disables the HTTP server (tests). Production + // default is :8080 — a dedicated port, not multiplexed onto gRPC :9093. + MetricsBindAddr string + + // ScrapeTimeout is the fan-out wait for MetricsStream responses (JEP default 7s). + ScrapeTimeout time.Duration + + // DriverTypeEnum allowlist; unknown driver_type values are remapped to "other". + DriverTypeEnum []string + + // ExemplarKeys allowlist applied on the telemetry merge path. + ExemplarKeys []string + + // Signer is used to validate bearer tokens on every PushLogs call and + // MetricsStream. Tokens are issued by the controller from the same + // CONTROLLER_KEY seed, so the telemetry binary can verify them locally + // without a k8s client. Signer *oidc.Signer + + // LokiConfig is used by Start to construct a LokiPusher when Loki is nil. + LokiConfig LokiConfig + + // Loki, when non-nil, forwards accepted PushLogs entries to Loki's HTTP API. + Loki *LokiPusher + + stateMu sync.Mutex + conns map[string]*metricsConn + scrapeTimeouts prometheus.Counter + parseErrors *prometheus.CounterVec + droppedTotal *prometheus.CounterVec + metricsRegistry *prometheus.Registry + metricsAddr string + grpcReady atomic.Bool } -// PushLogs receives a batch of structured log entries and writes them via the -// controller-runtime logger (structured JSON to stdout). -// Future phase: forward to Loki push API. +// PushLogs receives a batch of structured log entries, writes them via the +// controller-runtime logger (structured JSON to stdout), and enqueues them for +// Loki when a pusher is configured. func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest) (*pb.PushLogsResponse, error) { - token, err := authentication.BearerTokenFromContext(ctx) + id, err := s.authenticateExporter(ctx) if err != nil { return nil, err } - - // Validate token and extract the subject (format: exporter:namespace:name:uid). - subject, err := s.Signer.ParseSubject(token) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "invalid token: %v", err) - } - - // Only exporter tokens are allowed to push logs. Any other validly-signed - // token (e.g. a client token) is rejected immediately so that the identity - // checks below always have a non-empty claimedName/claimedNamespace. - parts := strings.SplitN(subject, ":", 4) - if len(parts) != 4 || parts[0] != "exporter" { - return nil, status.Errorf(codes.PermissionDenied, "token is not an exporter token") - } - claimedNamespace := parts[1] - claimedName := parts[2] - if claimedNamespace == "" || claimedName == "" { - return nil, status.Errorf(codes.PermissionDenied, "token has incomplete exporter identity") - } + claimedNamespace := id.namespace + claimedName := id.name // Use context-based logger so tests can inject their own via logf.IntoContext. logger := log.FromContext(ctx).WithName("telemetry") @@ -130,56 +143,49 @@ func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest continue } + prepared := prepareLogEntry(id, entry) + // Always log the authenticated identity. After the mismatch checks // above, any non-empty entry fields already match the token; using // the token values makes the server the source of truth for Loki // stream labels even when the entry omitted them. kvs := []any{ - "component", entry.Component, + "component", prepared.Component, logFieldExporter, claimedName, "namespace", claimedNamespace, - "severity", entry.Severity, + "severity", prepared.Severity, } - if entry.Timestamp != nil { - kvs = append(kvs, "ts", entry.Timestamp.AsTime().Format(time.RFC3339Nano)) + if prepared.Timestamp != nil { + kvs = append(kvs, "ts", prepared.Timestamp.AsTime().Format(time.RFC3339Nano)) } - if entry.Lease != "" { - kvs = append(kvs, "lease", entry.Lease) + if prepared.Lease != "" { + kvs = append(kvs, "lease", prepared.Lease) } - if entry.Client != "" { - kvs = append(kvs, "client", entry.Client) + if prepared.Client != "" { + kvs = append(kvs, "client", prepared.Client) } - if entry.Operation != "" { - kvs = append(kvs, "operation", entry.Operation) + if prepared.Operation != "" { + kvs = append(kvs, "operation", prepared.Operation) } - if entry.Result != "" { - kvs = append(kvs, "result", entry.Result) + if prepared.Result != "" { + kvs = append(kvs, "result", prepared.Result) } - if entry.DriverType != "" { - kvs = append(kvs, "driver_type", entry.DriverType) + if prepared.DriverType != "" { + kvs = append(kvs, "driver_type", prepared.DriverType) } - // Enforce extra_fields limits and strip reserved keys so an exporter - // cannot shadow trusted fields in downstream log parsers. - count := 0 - for k, v := range entry.ExtraFields { - if count >= maxExtraFields { - break - } - if _, reserved := reservedExtraFieldKeys[k]; reserved { - continue - } - k = truncate(k, maxKeyLen) - v = truncate(v, maxValueLen) + for k, v := range prepared.ExtraFields { kvs = append(kvs, k, v) - count++ } - switch strings.ToLower(entry.Severity) { + switch strings.ToLower(prepared.Severity) { case "error", "critical": - logger.Error(nil, entry.Message, kvs...) + logger.Error(nil, prepared.Message, kvs...) default: - logger.Info(entry.Message, kvs...) + logger.Info(prepared.Message, kvs...) + } + if s.Loki != nil { + s.Loki.Enqueue(prepared) } accepted++ } @@ -190,6 +196,43 @@ func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest }, nil } +// prepareLogEntry copies entry with identity overwritten from the token and +// extra_fields truncated / stripped of reserved keys. +func prepareLogEntry(id exporterIdentity, entry *pb.LogEntry) *pb.LogEntry { + out := &pb.LogEntry{ + Timestamp: entry.Timestamp, + Severity: entry.Severity, + Message: entry.Message, + Component: entry.Component, + Exporter: id.name, + Lease: entry.Lease, + Client: entry.Client, + Operation: entry.Operation, + Result: entry.Result, + DriverType: entry.DriverType, + Namespace: id.namespace, + } + if len(entry.ExtraFields) == 0 { + return out + } + extra := make(map[string]string, len(entry.ExtraFields)) + count := 0 + for k, v := range entry.ExtraFields { + if count >= maxExtraFields { + break + } + if _, reserved := reservedExtraFieldKeys[k]; reserved { + continue + } + extra[truncate(k, maxKeyLen)] = truncate(v, maxValueLen) + count++ + } + if len(extra) > 0 { + out.ExtraFields = extra + } + return out +} + // truncate returns s truncated to at most n bytes (rune-safe: truncates at rune boundary). func truncate(s string, n int) string { if len(s) <= n { @@ -260,9 +303,33 @@ func (s *TelemetryService) Start(ctx context.Context) error { return fmt.Errorf("telemetry: listen %s: %w", s.BindAddr, err) } + s.initMetricsState() + s.initScrapeTimeouts() + s.grpcReady.Store(true) + + if s.Loki == nil { + pusher, lokiErr := NewLokiPusher(s.LokiConfig, s.droppedTotal) + if lokiErr != nil { + logger.Error(lokiErr, "Loki push disabled") + } else { + s.Loki = pusher + } + } + if s.Loki != nil { + go s.Loki.Run(ctx) + } + + httpShutdown, err := s.startMetricsHTTP() + if err != nil { + s.grpcReady.Store(false) + _ = lis.Close() + return fmt.Errorf("telemetry: metrics HTTP listen %s: %w", s.MetricsBindAddr, err) + } + srv := grpc.NewServer( grpc.Creds(creds), grpc.ChainUnaryInterceptor(recovery.UnaryServerInterceptor()), + grpc.ChainStreamInterceptor(recovery.StreamServerInterceptor()), ) pb.RegisterTelemetryServiceServer(srv, s) reflection.Register(srv) @@ -276,12 +343,24 @@ func (s *TelemetryService) Start(ctx context.Context) error { select { case <-ctx.Done(): + s.grpcReady.Store(false) + if httpShutdown != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = httpShutdown(shutdownCtx) + } srv.GracefulStop() if err := <-errCh; err != nil && !errors.Is(err, grpc.ErrServerStopped) { return err } return nil case err := <-errCh: + s.grpcReady.Store(false) + if httpShutdown != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = httpShutdown(shutdownCtx) + } srv.Stop() return err } diff --git a/protocol/proto/jumpstarter/v1/telemetry.proto b/protocol/proto/jumpstarter/v1/telemetry.proto index 200748201..fbbfd00ee 100644 --- a/protocol/proto/jumpstarter/v1/telemetry.proto +++ b/protocol/proto/jumpstarter/v1/telemetry.proto @@ -6,13 +6,44 @@ package jumpstarter.v1; import "google/protobuf/timestamp.proto"; -// A service that receives structured logs from exporters and clients. +// A service that reverse-scrapes exporter metrics and receives structured logs. // Implemented by jumpstarter-telemetry; not part of the controller. service TelemetryService { + // Persistent bidirectional stream: telemetry sends scrape requests, + // exporter responds with full metric snapshots (OpenMetrics text). + rpc MetricsStream(stream MetricsStreamRequest) returns (stream MetricsStreamResponse); + // Push structured log entries to the telemetry service for Loki ingest. rpc PushLogs(PushLogsRequest) returns (PushLogsResponse); } +// Exporter → Telemetry +message MetricsStreamRequest { + oneof msg { + MetricsRegister register = 1; // First message: identify this exporter. + MetricsScrapeResponse scrape_response = 2; // Subsequent: reply to a scrape. + } +} + +message MetricsRegister { + string identity = 1; // Exporter CRD name (verified against the auth token by the server). +} + +message MetricsScrapeResponse { + bytes metrics_text = 1; // generate_latest() OpenMetrics output. + google.protobuf.Timestamp timestamp = 2; +} + +// Telemetry → Exporter +message MetricsStreamResponse { + oneof msg { + MetricsScrapeRequest scrape_request = 1; + } +} + +// Empty request: "send your /metrics now". +message MetricsScrapeRequest {} + // Request to push log entries to the telemetry service. message PushLogsRequest { repeated LogEntry entries = 1; // Log entries to push. diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py index 30983ccc3..80e4bc2bc 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py @@ -25,24 +25,34 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x0fPushLogsRequest\x122\n\x07entries\x18\x01 \x03(\x0b2\x18.jumpstarter.v1.LogEntryR\x07entries"H\n\x10PushLogsResponse\x12\x1a\n\x08accepted\x18\x01 \x01(\rR\x08accepted\x12\x18\n\x07dropped\x18\x02 \x01(\rR\x07dropped"\xe5\x03\n\x08LogEntry\x128\n\ttimestamp\x18\x01 \x01(\x0b2\x1a.google.protobuf.TimestampR\ttimestamp\x12\x1a\n\x08severity\x18\x02 \x01(\tR\x08severity\x12\x18\n\x07message\x18\x03 \x01(\tR\x07message\x12\x1c\n\tcomponent\x18\x04 \x01(\tR\tcomponent\x12\x1a\n\x08exporter\x18\x05 \x01(\tR\x08exporter\x12\x14\n\x05lease\x18\x06 \x01(\tR\x05lease\x12\x16\n\x06client\x18\x07 \x01(\tR\x06client\x12\x1c\n\toperation\x18\x08 \x01(\tR\toperation\x12\x16\n\x06result\x18\t \x01(\tR\x06result\x12\x1f\n\x0bdriver_type\x18\n \x01(\tR\ndriverType\x12L\n\x0cextra_fields\x18\x0b \x03(\x0b2).jumpstarter.v1.LogEntry.ExtraFieldsEntryR\x0bextraFields\x12\x1c\n\tnamespace\x18\x0c \x01(\tR\tnamespace\x1a>\n\x10ExtraFieldsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x028\x012a\n\x10TelemetryService\x12M\n\x08PushLogs\x12\x1f.jumpstarter.v1.PushLogsRequest\x1a .jumpstarter.v1.PushLogsResponseB\xd1\x01\n\x12com.jumpstarter.v1B\x0eTelemetryProtoP\x01ZRgithub.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1\xa2\x02\x03JXX\xaa\x02\x0eJumpstarter.V1\xca\x02\x0eJumpstarter\\V1\xe2\x02\x1aJumpstarter\\V1\\GPBMetadata\xea\x02\x0fJumpstarter::V1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xae\x01\n\x14MetricsStreamRequest\x12=\n\x08register\x18\x01 \x01(\x0b\x32\x1f.jumpstarter.v1.MetricsRegisterH\x00R\x08register\x12P\n\x0fscrape_response\x18\x02 \x01(\x0b\x32%.jumpstarter.v1.MetricsScrapeResponseH\x00R\x0escrapeResponseB\x05\n\x03msg\"-\n\x0fMetricsRegister\x12\x1a\n\x08identity\x18\x01 \x01(\tR\x08identity\"t\n\x15MetricsScrapeResponse\x12!\n\x0cmetrics_text\x18\x01 \x01(\x0cR\x0bmetricsText\x12\x38\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ttimestamp\"m\n\x15MetricsStreamResponse\x12M\n\x0escrape_request\x18\x01 \x01(\x0b\x32$.jumpstarter.v1.MetricsScrapeRequestH\x00R\rscrapeRequestB\x05\n\x03msg\"\x16\n\x14MetricsScrapeRequest\"E\n\x0fPushLogsRequest\x12\x32\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x18.jumpstarter.v1.LogEntryR\x07\x65ntries\"H\n\x10PushLogsResponse\x12\x1a\n\x08\x61\x63\x63\x65pted\x18\x01 \x01(\rR\x08\x61\x63\x63\x65pted\x12\x18\n\x07\x64ropped\x18\x02 \x01(\rR\x07\x64ropped\"\xe5\x03\n\x08LogEntry\x12\x38\n\ttimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ttimestamp\x12\x1a\n\x08severity\x18\x02 \x01(\tR\x08severity\x12\x18\n\x07message\x18\x03 \x01(\tR\x07message\x12\x1c\n\tcomponent\x18\x04 \x01(\tR\tcomponent\x12\x1a\n\x08\x65xporter\x18\x05 \x01(\tR\x08\x65xporter\x12\x14\n\x05lease\x18\x06 \x01(\tR\x05lease\x12\x16\n\x06\x63lient\x18\x07 \x01(\tR\x06\x63lient\x12\x1c\n\toperation\x18\x08 \x01(\tR\toperation\x12\x16\n\x06result\x18\t \x01(\tR\x06result\x12\x1f\n\x0b\x64river_type\x18\n \x01(\tR\ndriverType\x12L\n\x0c\x65xtra_fields\x18\x0b \x03(\x0b\x32).jumpstarter.v1.LogEntry.ExtraFieldsEntryR\x0b\x65xtraFields\x12\x1c\n\tnamespace\x18\x0c \x01(\tR\tnamespace\x1a>\n\x10\x45xtraFieldsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x32\xc3\x01\n\x10TelemetryService\x12`\n\rMetricsStream\x12$.jumpstarter.v1.MetricsStreamRequest\x1a%.jumpstarter.v1.MetricsStreamResponse(\x01\x30\x01\x12M\n\x08PushLogs\x12\x1f.jumpstarter.v1.PushLogsRequest\x1a .jumpstarter.v1.PushLogsResponseB}\n\x12\x63om.jumpstarter.v1B\x0eTelemetryProtoP\x01\xa2\x02\x03JXX\xaa\x02\x0eJumpstarter.V1\xca\x02\x0eJumpstarter\\V1\xe2\x02\x1aJumpstarter\\V1\\GPBMetadata\xea\x02\x0fJumpstarter::V1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'jumpstarter.v1.telemetry_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\022com.jumpstarter.v1B\016TelemetryProtoP\001ZRgithub.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1\242\002\003JXX\252\002\016Jumpstarter.V1\312\002\016Jumpstarter\\V1\342\002\032Jumpstarter\\V1\\GPBMetadata\352\002\017Jumpstarter::V1' + _globals['DESCRIPTOR']._serialized_options = b'\n\022com.jumpstarter.v1B\016TelemetryProtoP\001\242\002\003JXX\252\002\016Jumpstarter.V1\312\002\016Jumpstarter\\V1\342\002\032Jumpstarter\\V1\\GPBMetadata\352\002\017Jumpstarter::V1' _globals['_LOGENTRY_EXTRAFIELDSENTRY']._loaded_options = None _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_options = b'8\001' - _globals['_PUSHLOGSREQUEST']._serialized_start=83 - _globals['_PUSHLOGSREQUEST']._serialized_end=152 - _globals['_PUSHLOGSRESPONSE']._serialized_start=154 - _globals['_PUSHLOGSRESPONSE']._serialized_end=226 - _globals['_LOGENTRY']._serialized_start=229 - _globals['_LOGENTRY']._serialized_end=714 - _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_start=652 - _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_end=714 - _globals['_TELEMETRYSERVICE']._serialized_start=716 - _globals['_TELEMETRYSERVICE']._serialized_end=813 + _globals['_METRICSSTREAMREQUEST']._serialized_start=84 + _globals['_METRICSSTREAMREQUEST']._serialized_end=258 + _globals['_METRICSREGISTER']._serialized_start=260 + _globals['_METRICSREGISTER']._serialized_end=305 + _globals['_METRICSSCRAPERESPONSE']._serialized_start=307 + _globals['_METRICSSCRAPERESPONSE']._serialized_end=423 + _globals['_METRICSSTREAMRESPONSE']._serialized_start=425 + _globals['_METRICSSTREAMRESPONSE']._serialized_end=534 + _globals['_METRICSSCRAPEREQUEST']._serialized_start=536 + _globals['_METRICSSCRAPEREQUEST']._serialized_end=558 + _globals['_PUSHLOGSREQUEST']._serialized_start=560 + _globals['_PUSHLOGSREQUEST']._serialized_end=629 + _globals['_PUSHLOGSRESPONSE']._serialized_start=631 + _globals['_PUSHLOGSRESPONSE']._serialized_end=703 + _globals['_LOGENTRY']._serialized_start=706 + _globals['_LOGENTRY']._serialized_end=1191 + _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_start=1129 + _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_end=1191 + _globals['_TELEMETRYSERVICE']._serialized_start=1194 + _globals['_TELEMETRYSERVICE']._serialized_end=1389 # @@protoc_insertion_point(module_scope) diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi index 2f28d9f24..968f7ea75 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi @@ -19,6 +19,103 @@ else: DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +@typing.final +class MetricsStreamRequest(google.protobuf.message.Message): + """Exporter → Telemetry""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REGISTER_FIELD_NUMBER: builtins.int + SCRAPE_RESPONSE_FIELD_NUMBER: builtins.int + @property + def register(self) -> Global___MetricsRegister: + """First message: identify this exporter.""" + + @property + def scrape_response(self) -> Global___MetricsScrapeResponse: + """Subsequent: reply to a scrape.""" + + def __init__( + self, + *, + register: Global___MetricsRegister | None = ..., + scrape_response: Global___MetricsScrapeResponse | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["msg", b"msg", "register", b"register", "scrape_response", b"scrape_response"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["msg", b"msg", "register", b"register", "scrape_response", b"scrape_response"]) -> None: ... + def WhichOneof(self, oneof_group: typing.Literal["msg", b"msg"]) -> typing.Literal["register", "scrape_response"] | None: ... + +Global___MetricsStreamRequest: typing_extensions.TypeAlias = MetricsStreamRequest + +@typing.final +class MetricsRegister(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + IDENTITY_FIELD_NUMBER: builtins.int + identity: builtins.str + """Exporter CRD name (verified against the auth token by the server).""" + def __init__( + self, + *, + identity: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["identity", b"identity"]) -> None: ... + +Global___MetricsRegister: typing_extensions.TypeAlias = MetricsRegister + +@typing.final +class MetricsScrapeResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + METRICS_TEXT_FIELD_NUMBER: builtins.int + TIMESTAMP_FIELD_NUMBER: builtins.int + metrics_text: builtins.bytes + """generate_latest() OpenMetrics output.""" + @property + def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + metrics_text: builtins.bytes = ..., + timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["timestamp", b"timestamp"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["metrics_text", b"metrics_text", "timestamp", b"timestamp"]) -> None: ... + +Global___MetricsScrapeResponse: typing_extensions.TypeAlias = MetricsScrapeResponse + +@typing.final +class MetricsStreamResponse(google.protobuf.message.Message): + """Telemetry → Exporter""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SCRAPE_REQUEST_FIELD_NUMBER: builtins.int + @property + def scrape_request(self) -> Global___MetricsScrapeRequest: ... + def __init__( + self, + *, + scrape_request: Global___MetricsScrapeRequest | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["msg", b"msg", "scrape_request", b"scrape_request"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["msg", b"msg", "scrape_request", b"scrape_request"]) -> None: ... + def WhichOneof(self, oneof_group: typing.Literal["msg", b"msg"]) -> typing.Literal["scrape_request"] | None: ... + +Global___MetricsStreamResponse: typing_extensions.TypeAlias = MetricsStreamResponse + +@typing.final +class MetricsScrapeRequest(google.protobuf.message.Message): + """Empty request: "send your /metrics now".""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +Global___MetricsScrapeRequest: typing_extensions.TypeAlias = MetricsScrapeRequest + @typing.final class PushLogsRequest(google.protobuf.message.Message): """Request to push log entries to the telemetry service.""" @@ -96,6 +193,7 @@ class LogEntry(google.protobuf.message.Message): RESULT_FIELD_NUMBER: builtins.int DRIVER_TYPE_FIELD_NUMBER: builtins.int EXTRA_FIELDS_FIELD_NUMBER: builtins.int + NAMESPACE_FIELD_NUMBER: builtins.int severity: builtins.str """Log severity: debug, info, warning, error, critical.""" message: builtins.str @@ -115,7 +213,9 @@ class LogEntry(google.protobuf.message.Message): driver_type: builtins.str """Log body: driver category (storage, power, network, etc.).""" namespace: builtins.str - """Loki stream label: Kubernetes namespace (bounded by cluster size).""" + """Capped at 16 entries, 64-char keys, 256-char values. + Loki stream label: Kubernetes namespace (bounded by cluster size). + """ @property def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: """When the log was emitted.""" diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py index 7be5e24a0..e34c217b6 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py @@ -6,7 +6,7 @@ class TelemetryServiceStub: - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ @@ -16,6 +16,11 @@ def __init__(self, channel): Args: channel: A grpc.Channel. """ + self.MetricsStream = channel.stream_stream( + '/jumpstarter.v1.TelemetryService/MetricsStream', + request_serializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamRequest.SerializeToString, + response_deserializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamResponse.FromString, + _registered_method=True) self.PushLogs = channel.unary_unary( '/jumpstarter.v1.TelemetryService/PushLogs', request_serializer=jumpstarter_dot_v1_dot_telemetry__pb2.PushLogsRequest.SerializeToString, @@ -24,10 +29,18 @@ def __init__(self, channel): class TelemetryServiceServicer: - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ + def MetricsStream(self, request_iterator, context): + """Persistent bidirectional stream: telemetry sends scrape requests, + exporter responds with full metric snapshots (OpenMetrics text). + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def PushLogs(self, request, context): """Push structured log entries to the telemetry service for Loki ingest. """ @@ -38,6 +51,11 @@ def PushLogs(self, request, context): def add_TelemetryServiceServicer_to_server(servicer, server): rpc_method_handlers = { + 'MetricsStream': grpc.stream_stream_rpc_method_handler( + servicer.MetricsStream, + request_deserializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamRequest.FromString, + response_serializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamResponse.SerializeToString, + ), 'PushLogs': grpc.unary_unary_rpc_method_handler( servicer.PushLogs, request_deserializer=jumpstarter_dot_v1_dot_telemetry__pb2.PushLogsRequest.FromString, @@ -52,10 +70,37 @@ def add_TelemetryServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class TelemetryService: - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ + @staticmethod + def MetricsStream(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream( + request_iterator, + target, + '/jumpstarter.v1.TelemetryService/MetricsStream', + jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamRequest.SerializeToString, + jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def PushLogs(request, target, diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi index dc163b903..d810b4378 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi @@ -25,6 +25,22 @@ class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext): # type: GRPC_GENERATED_VERSION: str GRPC_VERSION: str +_TelemetryServiceMetricsStreamType = typing_extensions.TypeVar( + '_TelemetryServiceMetricsStreamType', + grpc.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], + grpc.aio.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], + default=grpc.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], +) + _TelemetryServicePushLogsType = typing_extensions.TypeVar( '_TelemetryServicePushLogsType', grpc.UnaryUnaryMultiCallable[ @@ -41,13 +57,17 @@ _TelemetryServicePushLogsType = typing_extensions.TypeVar( ], ) -class TelemetryServiceStub(typing.Generic[_TelemetryServicePushLogsType]): - """A service that receives structured logs from exporters and clients. +class TelemetryServiceStub(typing.Generic[_TelemetryServiceMetricsStreamType, _TelemetryServicePushLogsType]): + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ @typing.overload def __init__(self: TelemetryServiceStub[ + grpc.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], grpc.UnaryUnaryMultiCallable[ jumpstarter.v1.telemetry_pb2.PushLogsRequest, jumpstarter.v1.telemetry_pb2.PushLogsResponse, @@ -56,16 +76,29 @@ class TelemetryServiceStub(typing.Generic[_TelemetryServicePushLogsType]): @typing.overload def __init__(self: TelemetryServiceStub[ + grpc.aio.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], grpc.aio.UnaryUnaryMultiCallable[ jumpstarter.v1.telemetry_pb2.PushLogsRequest, jumpstarter.v1.telemetry_pb2.PushLogsResponse, ], ], channel: grpc.aio.Channel) -> None: ... + MetricsStream: _TelemetryServiceMetricsStreamType + """Persistent bidirectional stream: telemetry sends scrape requests, + exporter responds with full metric snapshots (OpenMetrics text). + """ + PushLogs: _TelemetryServicePushLogsType """Push structured log entries to the telemetry service for Loki ingest.""" TelemetryServiceAsyncStub: typing_extensions.TypeAlias = TelemetryServiceStub[ + grpc.aio.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], grpc.aio.UnaryUnaryMultiCallable[ jumpstarter.v1.telemetry_pb2.PushLogsRequest, jumpstarter.v1.telemetry_pb2.PushLogsResponse, @@ -73,10 +106,20 @@ TelemetryServiceAsyncStub: typing_extensions.TypeAlias = TelemetryServiceStub[ ] class TelemetryServiceServicer(metaclass=abc.ABCMeta): - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ + @abc.abstractmethod + def MetricsStream( + self, + request_iterator: _MaybeAsyncIterator[jumpstarter.v1.telemetry_pb2.MetricsStreamRequest], + context: _ServicerContext, + ) -> typing.Union[collections.abc.Iterator[jumpstarter.v1.telemetry_pb2.MetricsStreamResponse], collections.abc.AsyncIterator[jumpstarter.v1.telemetry_pb2.MetricsStreamResponse]]: + """Persistent bidirectional stream: telemetry sends scrape requests, + exporter responds with full metric snapshots (OpenMetrics text). + """ + @abc.abstractmethod def PushLogs( self,