From a2d749801eb56005c433f964afff4259ec97d56c Mon Sep 17 00:00:00 2001 From: Roddie Kieley Date: Wed, 26 Aug 2026 19:01:32 -0230 Subject: [PATCH 1/5] feat: telemetry MetricsStream reverse-scrape hub (JEP-0013 Phase 3) Add the MetricsStream protocol and Go hub so Prometheus can scrape merged exporter OpenMetrics from telemetry without an exporter client yet. Generated Python stubs are included for proto consistency. Co-authored-by: Cursor --- controller/cmd/telemetry/main.go | 49 ++- .../protocol/jumpstarter/v1/telemetry.pb.go | 402 ++++++++++++++++-- .../jumpstarter/v1/telemetry_grpc.pb.go | 47 +- controller/internal/service/metrics_merge.go | 239 +++++++++++ .../internal/service/metrics_merge_test.go | 172 ++++++++ controller/internal/service/metrics_stream.go | 241 +++++++++++ .../internal/service/metrics_stream_test.go | 402 ++++++++++++++++++ controller/internal/service/telemetry_http.go | 135 ++++++ .../internal/service/telemetry_identity.go | 62 +++ .../internal/service/telemetry_service.go | 83 ++-- protocol/proto/jumpstarter/v1/telemetry.proto | 33 +- .../jumpstarter/v1/telemetry_pb2.py | 34 +- .../jumpstarter/v1/telemetry_pb2.pyi | 102 ++++- .../jumpstarter/v1/telemetry_pb2_grpc.py | 51 ++- .../jumpstarter/v1/telemetry_pb2_grpc.pyi | 49 ++- 15 files changed, 1999 insertions(+), 102 deletions(-) create mode 100644 controller/internal/service/metrics_merge.go create mode 100644 controller/internal/service/metrics_merge_test.go create mode 100644 controller/internal/service/metrics_stream.go create mode 100644 controller/internal/service/metrics_stream_test.go create mode 100644 controller/internal/service/telemetry_http.go create mode 100644 controller/internal/service/telemetry_identity.go diff --git a/controller/cmd/telemetry/main.go b/controller/cmd/telemetry/main.go index 1c757d42e..cf2266048 100644 --- a/controller/cmd/telemetry/main.go +++ b/controller/cmd/telemetry/main.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// jumpstarter-telemetry receives structured log entries from exporters and clients -// via the PushLogs gRPC RPC and writes them to structured stdout for downstream -// log shippers (Promtail, Grafana Alloy, Vector) to forward to Loki. +// jumpstarter-telemetry reverse-scrapes exporter metrics via MetricsStream and +// receives structured log entries via PushLogs. Logs are written to structured +// stdout for downstream log shippers (Promtail, Grafana Alloy, Vector). +// Loki push is a later Phase 3 PR. // // TLS: always enabled. Set EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM to file paths of // operator-mounted cert/key (e.g. from a cert-manager Secret); when absent a @@ -30,8 +31,7 @@ limitations under the License. // certificate; the controller uses it to advertise the address to exporters via // GetServiceEndpoints. A mismatch causes TLS hostname verification failures. // -// Future phases will add direct Loki push and MetricsStream for reverse-scrape -// of exporter prometheus_client registries. +// HTTP: GET /metrics, /healthz, and /readyz bind separately (default :8080). package main import ( @@ -39,7 +39,9 @@ import ( "flag" "os" "os/signal" + "strings" "syscall" + "time" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -55,9 +57,36 @@ var ( buildDate = "unknown" ) +func splitCSV(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + func main() { var bindAddr string + var metricsAddr string + var scrapeTimeout time.Duration + var driverTypeEnum string + var exemplarKeys string flag.StringVar(&bindAddr, "grpc-bind", ":9093", "TCP address to bind the gRPC server to") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", + "TCP address for HTTP GET /metrics, /healthz, and /readyz. Use 0 to disable.") + flag.DurationVar(&scrapeTimeout, "scrape-timeout", 7*time.Second, + "Max wait for parallel MetricsStream scrape responses") + flag.StringVar(&driverTypeEnum, "driver-type-enum", strings.Join(service.DefaultDriverTypeEnum, ","), + "Comma-separated allowlist of driver_type values; others are remapped to other") + flag.StringVar(&exemplarKeys, "exemplar-keys", strings.Join(service.DefaultExemplarKeys, ","), + "Comma-separated allowlist of Prometheus exemplar keys") opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -71,6 +100,8 @@ func main() { "gitCommit", gitCommit, "buildDate", buildDate, "bindAddr", bindAddr, + "metricsBindAddr", metricsAddr, + "scrapeTimeout", scrapeTimeout, ) ctx, cancel := context.WithCancel(context.Background()) @@ -87,8 +118,12 @@ func main() { } svc := &service.TelemetryService{ - BindAddr: bindAddr, - Signer: signer, + BindAddr: bindAddr, + MetricsBindAddr: metricsAddr, + ScrapeTimeout: scrapeTimeout, + DriverTypeEnum: splitCSV(driverTypeEnum), + ExemplarKeys: splitCSV(exemplarKeys), + Signer: signer, } // Register signal handler before starting the service so no signal diff --git a/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go b/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go index 00e583612..ee0ac7352 100644 --- a/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go +++ b/controller/internal/protocol/jumpstarter/v1/telemetry.pb.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Jumpstarter Authors +// Copyright 2026 The Jumpstarter Authors // Code generated by protoc-gen-go. DO NOT EDIT. // versions: @@ -24,6 +24,289 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Exporter → Telemetry +type MetricsStreamRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Msg: + // + // *MetricsStreamRequest_Register + // *MetricsStreamRequest_ScrapeResponse + Msg isMetricsStreamRequest_Msg `protobuf_oneof:"msg"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsStreamRequest) Reset() { + *x = MetricsStreamRequest{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsStreamRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsStreamRequest) ProtoMessage() {} + +func (x *MetricsStreamRequest) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsStreamRequest.ProtoReflect.Descriptor instead. +func (*MetricsStreamRequest) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{0} +} + +func (x *MetricsStreamRequest) GetMsg() isMetricsStreamRequest_Msg { + if x != nil { + return x.Msg + } + return nil +} + +func (x *MetricsStreamRequest) GetRegister() *MetricsRegister { + if x != nil { + if x, ok := x.Msg.(*MetricsStreamRequest_Register); ok { + return x.Register + } + } + return nil +} + +func (x *MetricsStreamRequest) GetScrapeResponse() *MetricsScrapeResponse { + if x != nil { + if x, ok := x.Msg.(*MetricsStreamRequest_ScrapeResponse); ok { + return x.ScrapeResponse + } + } + return nil +} + +type isMetricsStreamRequest_Msg interface { + isMetricsStreamRequest_Msg() +} + +type MetricsStreamRequest_Register struct { + Register *MetricsRegister `protobuf:"bytes,1,opt,name=register,proto3,oneof"` // First message: identify this exporter. +} + +type MetricsStreamRequest_ScrapeResponse struct { + ScrapeResponse *MetricsScrapeResponse `protobuf:"bytes,2,opt,name=scrape_response,json=scrapeResponse,proto3,oneof"` // Subsequent: reply to a scrape. +} + +func (*MetricsStreamRequest_Register) isMetricsStreamRequest_Msg() {} + +func (*MetricsStreamRequest_ScrapeResponse) isMetricsStreamRequest_Msg() {} + +type MetricsRegister struct { + state protoimpl.MessageState `protogen:"open.v1"` + Identity string `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"` // Exporter CRD name (verified against the auth token by the server). + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsRegister) Reset() { + *x = MetricsRegister{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsRegister) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsRegister) ProtoMessage() {} + +func (x *MetricsRegister) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsRegister.ProtoReflect.Descriptor instead. +func (*MetricsRegister) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{1} +} + +func (x *MetricsRegister) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + +type MetricsScrapeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MetricsText []byte `protobuf:"bytes,1,opt,name=metrics_text,json=metricsText,proto3" json:"metrics_text,omitempty"` // generate_latest() OpenMetrics output. + Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsScrapeResponse) Reset() { + *x = MetricsScrapeResponse{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsScrapeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsScrapeResponse) ProtoMessage() {} + +func (x *MetricsScrapeResponse) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsScrapeResponse.ProtoReflect.Descriptor instead. +func (*MetricsScrapeResponse) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{2} +} + +func (x *MetricsScrapeResponse) GetMetricsText() []byte { + if x != nil { + return x.MetricsText + } + return nil +} + +func (x *MetricsScrapeResponse) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +// Telemetry → Exporter +type MetricsStreamResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Msg: + // + // *MetricsStreamResponse_ScrapeRequest + Msg isMetricsStreamResponse_Msg `protobuf_oneof:"msg"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsStreamResponse) Reset() { + *x = MetricsStreamResponse{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsStreamResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsStreamResponse) ProtoMessage() {} + +func (x *MetricsStreamResponse) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsStreamResponse.ProtoReflect.Descriptor instead. +func (*MetricsStreamResponse) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{3} +} + +func (x *MetricsStreamResponse) GetMsg() isMetricsStreamResponse_Msg { + if x != nil { + return x.Msg + } + return nil +} + +func (x *MetricsStreamResponse) GetScrapeRequest() *MetricsScrapeRequest { + if x != nil { + if x, ok := x.Msg.(*MetricsStreamResponse_ScrapeRequest); ok { + return x.ScrapeRequest + } + } + return nil +} + +type isMetricsStreamResponse_Msg interface { + isMetricsStreamResponse_Msg() +} + +type MetricsStreamResponse_ScrapeRequest struct { + ScrapeRequest *MetricsScrapeRequest `protobuf:"bytes,1,opt,name=scrape_request,json=scrapeRequest,proto3,oneof"` +} + +func (*MetricsStreamResponse_ScrapeRequest) isMetricsStreamResponse_Msg() {} + +// Empty request: "send your /metrics now". +type MetricsScrapeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MetricsScrapeRequest) Reset() { + *x = MetricsScrapeRequest{} + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MetricsScrapeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsScrapeRequest) ProtoMessage() {} + +func (x *MetricsScrapeRequest) ProtoReflect() protoreflect.Message { + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsScrapeRequest.ProtoReflect.Descriptor instead. +func (*MetricsScrapeRequest) Descriptor() ([]byte, []int) { + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{4} +} + // Request to push log entries to the telemetry service. type PushLogsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -34,7 +317,7 @@ type PushLogsRequest struct { func (x *PushLogsRequest) Reset() { *x = PushLogsRequest{} - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -46,7 +329,7 @@ func (x *PushLogsRequest) String() string { func (*PushLogsRequest) ProtoMessage() {} func (x *PushLogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[0] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -59,7 +342,7 @@ func (x *PushLogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogsRequest.ProtoReflect.Descriptor instead. func (*PushLogsRequest) Descriptor() ([]byte, []int) { - return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{0} + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{5} } func (x *PushLogsRequest) GetEntries() []*LogEntry { @@ -80,7 +363,7 @@ type PushLogsResponse struct { func (x *PushLogsResponse) Reset() { *x = PushLogsResponse{} - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -92,7 +375,7 @@ func (x *PushLogsResponse) String() string { func (*PushLogsResponse) ProtoMessage() {} func (x *PushLogsResponse) ProtoReflect() protoreflect.Message { - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[1] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -105,7 +388,7 @@ func (x *PushLogsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PushLogsResponse.ProtoReflect.Descriptor instead. func (*PushLogsResponse) Descriptor() ([]byte, []int) { - return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{1} + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{6} } func (x *PushLogsResponse) GetAccepted() uint32 { @@ -125,26 +408,27 @@ func (x *PushLogsResponse) GetDropped() uint32 { // A structured log entry from an exporter or client. // Maps directly to a Loki log entry with stream labels and body fields. type LogEntry struct { - state protoimpl.MessageState `protogen:"open.v1"` - Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // When the log was emitted. - Severity string `protobuf:"bytes,2,opt,name=severity,proto3" json:"severity,omitempty"` // Log severity: debug, info, warning, error, critical. - Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"` // Human-readable log message. - Component string `protobuf:"bytes,4,opt,name=component,proto3" json:"component,omitempty"` // Loki stream label: cli, exporter, controller, router, telemetry. - Exporter string `protobuf:"bytes,5,opt,name=exporter,proto3" json:"exporter,omitempty"` // Loki stream label: exporter CRD name (bounded by cluster size). - Lease string `protobuf:"bytes,6,opt,name=lease,proto3" json:"lease,omitempty"` // Log body only (high cardinality): active lease name. - Client string `protobuf:"bytes,7,opt,name=client,proto3" json:"client,omitempty"` // Log body only (high cardinality): client CRD name. - Operation string `protobuf:"bytes,8,opt,name=operation,proto3" json:"operation,omitempty"` // Log body: operation name (flash, power, etc.). - Result string `protobuf:"bytes,9,opt,name=result,proto3" json:"result,omitempty"` // Log body: operation outcome (success, failure, etc.). - DriverType string `protobuf:"bytes,10,opt,name=driver_type,json=driverType,proto3" json:"driver_type,omitempty"` // Log body: driver category (storage, power, network, etc.). - ExtraFields map[string]string `protobuf:"bytes,11,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Additional structured key-value fields. - Namespace string `protobuf:"bytes,12,opt,name=namespace,proto3" json:"namespace,omitempty"` // Loki stream label: Kubernetes namespace (bounded by cluster size). + state protoimpl.MessageState `protogen:"open.v1"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // When the log was emitted. + Severity string `protobuf:"bytes,2,opt,name=severity,proto3" json:"severity,omitempty"` // Log severity: debug, info, warning, error, critical. + Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"` // Human-readable log message. + Component string `protobuf:"bytes,4,opt,name=component,proto3" json:"component,omitempty"` // Loki stream label: cli, exporter, controller, router, telemetry. + Exporter string `protobuf:"bytes,5,opt,name=exporter,proto3" json:"exporter,omitempty"` // Loki stream label: exporter CRD name (bounded by cluster size). + Lease string `protobuf:"bytes,6,opt,name=lease,proto3" json:"lease,omitempty"` // Log body only (high cardinality): active lease name. + Client string `protobuf:"bytes,7,opt,name=client,proto3" json:"client,omitempty"` // Log body only (high cardinality): client CRD name. + Operation string `protobuf:"bytes,8,opt,name=operation,proto3" json:"operation,omitempty"` // Log body: operation name (flash, power, etc.). + Result string `protobuf:"bytes,9,opt,name=result,proto3" json:"result,omitempty"` // Log body: operation outcome (success, failure, etc.). + DriverType string `protobuf:"bytes,10,opt,name=driver_type,json=driverType,proto3" json:"driver_type,omitempty"` // Log body: driver category (storage, power, network, etc.). + ExtraFields map[string]string `protobuf:"bytes,11,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Additional structured key-value fields. + // Capped at 16 entries, 64-char keys, 256-char values. + Namespace string `protobuf:"bytes,12,opt,name=namespace,proto3" json:"namespace,omitempty"` // Loki stream label: Kubernetes namespace (bounded by cluster size). unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *LogEntry) Reset() { *x = LogEntry{} - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -156,7 +440,7 @@ func (x *LogEntry) String() string { func (*LogEntry) ProtoMessage() {} func (x *LogEntry) ProtoReflect() protoreflect.Message { - mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[2] + mi := &file_jumpstarter_v1_telemetry_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -169,7 +453,7 @@ func (x *LogEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. func (*LogEntry) Descriptor() ([]byte, []int) { - return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{2} + return file_jumpstarter_v1_telemetry_proto_rawDescGZIP(), []int{7} } func (x *LogEntry) GetTimestamp() *timestamppb.Timestamp { @@ -260,7 +544,20 @@ var File_jumpstarter_v1_telemetry_proto protoreflect.FileDescriptor const file_jumpstarter_v1_telemetry_proto_rawDesc = "" + "\n" + - "\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"E\n" + + "\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xae\x01\n" + + "\x14MetricsStreamRequest\x12=\n" + + "\bregister\x18\x01 \x01(\v2\x1f.jumpstarter.v1.MetricsRegisterH\x00R\bregister\x12P\n" + + "\x0fscrape_response\x18\x02 \x01(\v2%.jumpstarter.v1.MetricsScrapeResponseH\x00R\x0escrapeResponseB\x05\n" + + "\x03msg\"-\n" + + "\x0fMetricsRegister\x12\x1a\n" + + "\bidentity\x18\x01 \x01(\tR\bidentity\"t\n" + + "\x15MetricsScrapeResponse\x12!\n" + + "\fmetrics_text\x18\x01 \x01(\fR\vmetricsText\x128\n" + + "\ttimestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"m\n" + + "\x15MetricsStreamResponse\x12M\n" + + "\x0escrape_request\x18\x01 \x01(\v2$.jumpstarter.v1.MetricsScrapeRequestH\x00R\rscrapeRequestB\x05\n" + + "\x03msg\"\x16\n" + + "\x14MetricsScrapeRequest\"E\n" + "\x0fPushLogsRequest\x122\n" + "\aentries\x18\x01 \x03(\v2\x18.jumpstarter.v1.LogEntryR\aentries\"H\n" + "\x10PushLogsResponse\x12\x1a\n" + @@ -283,8 +580,9 @@ const file_jumpstarter_v1_telemetry_proto_rawDesc = "" + "\tnamespace\x18\f \x01(\tR\tnamespace\x1a>\n" + "\x10ExtraFieldsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x012a\n" + - "\x10TelemetryService\x12M\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x012\xc3\x01\n" + + "\x10TelemetryService\x12`\n" + + "\rMetricsStream\x12$.jumpstarter.v1.MetricsStreamRequest\x1a%.jumpstarter.v1.MetricsStreamResponse(\x010\x01\x12M\n" + "\bPushLogs\x12\x1f.jumpstarter.v1.PushLogsRequest\x1a .jumpstarter.v1.PushLogsResponseB\xdf\x01\n" + "\x12com.jumpstarter.v1B\x0eTelemetryProtoP\x01Z`github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1;jumpstarterv1\xa2\x02\x03JXX\xaa\x02\x0eJumpstarter.V1\xca\x02\x0eJumpstarter\\V1\xe2\x02\x1aJumpstarter\\V1\\GPBMetadata\xea\x02\x0fJumpstarter::V1b\x06proto3" @@ -300,25 +598,36 @@ func file_jumpstarter_v1_telemetry_proto_rawDescGZIP() []byte { return file_jumpstarter_v1_telemetry_proto_rawDescData } -var file_jumpstarter_v1_telemetry_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_jumpstarter_v1_telemetry_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_jumpstarter_v1_telemetry_proto_goTypes = []any{ - (*PushLogsRequest)(nil), // 0: jumpstarter.v1.PushLogsRequest - (*PushLogsResponse)(nil), // 1: jumpstarter.v1.PushLogsResponse - (*LogEntry)(nil), // 2: jumpstarter.v1.LogEntry - nil, // 3: jumpstarter.v1.LogEntry.ExtraFieldsEntry - (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp + (*MetricsStreamRequest)(nil), // 0: jumpstarter.v1.MetricsStreamRequest + (*MetricsRegister)(nil), // 1: jumpstarter.v1.MetricsRegister + (*MetricsScrapeResponse)(nil), // 2: jumpstarter.v1.MetricsScrapeResponse + (*MetricsStreamResponse)(nil), // 3: jumpstarter.v1.MetricsStreamResponse + (*MetricsScrapeRequest)(nil), // 4: jumpstarter.v1.MetricsScrapeRequest + (*PushLogsRequest)(nil), // 5: jumpstarter.v1.PushLogsRequest + (*PushLogsResponse)(nil), // 6: jumpstarter.v1.PushLogsResponse + (*LogEntry)(nil), // 7: jumpstarter.v1.LogEntry + nil, // 8: jumpstarter.v1.LogEntry.ExtraFieldsEntry + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp } var file_jumpstarter_v1_telemetry_proto_depIdxs = []int32{ - 2, // 0: jumpstarter.v1.PushLogsRequest.entries:type_name -> jumpstarter.v1.LogEntry - 4, // 1: jumpstarter.v1.LogEntry.timestamp:type_name -> google.protobuf.Timestamp - 3, // 2: jumpstarter.v1.LogEntry.extra_fields:type_name -> jumpstarter.v1.LogEntry.ExtraFieldsEntry - 0, // 3: jumpstarter.v1.TelemetryService.PushLogs:input_type -> jumpstarter.v1.PushLogsRequest - 1, // 4: jumpstarter.v1.TelemetryService.PushLogs:output_type -> jumpstarter.v1.PushLogsResponse - 4, // [4:5] is the sub-list for method output_type - 3, // [3:4] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 1, // 0: jumpstarter.v1.MetricsStreamRequest.register:type_name -> jumpstarter.v1.MetricsRegister + 2, // 1: jumpstarter.v1.MetricsStreamRequest.scrape_response:type_name -> jumpstarter.v1.MetricsScrapeResponse + 9, // 2: jumpstarter.v1.MetricsScrapeResponse.timestamp:type_name -> google.protobuf.Timestamp + 4, // 3: jumpstarter.v1.MetricsStreamResponse.scrape_request:type_name -> jumpstarter.v1.MetricsScrapeRequest + 7, // 4: jumpstarter.v1.PushLogsRequest.entries:type_name -> jumpstarter.v1.LogEntry + 9, // 5: jumpstarter.v1.LogEntry.timestamp:type_name -> google.protobuf.Timestamp + 8, // 6: jumpstarter.v1.LogEntry.extra_fields:type_name -> jumpstarter.v1.LogEntry.ExtraFieldsEntry + 0, // 7: jumpstarter.v1.TelemetryService.MetricsStream:input_type -> jumpstarter.v1.MetricsStreamRequest + 5, // 8: jumpstarter.v1.TelemetryService.PushLogs:input_type -> jumpstarter.v1.PushLogsRequest + 3, // 9: jumpstarter.v1.TelemetryService.MetricsStream:output_type -> jumpstarter.v1.MetricsStreamResponse + 6, // 10: jumpstarter.v1.TelemetryService.PushLogs:output_type -> jumpstarter.v1.PushLogsResponse + 9, // [9:11] is the sub-list for method output_type + 7, // [7:9] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_jumpstarter_v1_telemetry_proto_init() } @@ -326,13 +635,20 @@ func file_jumpstarter_v1_telemetry_proto_init() { if File_jumpstarter_v1_telemetry_proto != nil { return } + file_jumpstarter_v1_telemetry_proto_msgTypes[0].OneofWrappers = []any{ + (*MetricsStreamRequest_Register)(nil), + (*MetricsStreamRequest_ScrapeResponse)(nil), + } + file_jumpstarter_v1_telemetry_proto_msgTypes[3].OneofWrappers = []any{ + (*MetricsStreamResponse_ScrapeRequest)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_jumpstarter_v1_telemetry_proto_rawDesc), len(file_jumpstarter_v1_telemetry_proto_rawDesc)), NumEnums: 0, - NumMessages: 4, + NumMessages: 9, NumExtensions: 0, NumServices: 1, }, diff --git a/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go b/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go index 356b656a2..d5db4a9a7 100644 --- a/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go +++ b/controller/internal/protocol/jumpstarter/v1/telemetry_grpc.pb.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Jumpstarter Authors +// Copyright 2026 The Jumpstarter Authors // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: @@ -21,16 +21,20 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - TelemetryService_PushLogs_FullMethodName = "/jumpstarter.v1.TelemetryService/PushLogs" + TelemetryService_MetricsStream_FullMethodName = "/jumpstarter.v1.TelemetryService/MetricsStream" + TelemetryService_PushLogs_FullMethodName = "/jumpstarter.v1.TelemetryService/PushLogs" ) // TelemetryServiceClient is the client API for TelemetryService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// A service that receives structured logs from exporters and clients. +// A service that reverse-scrapes exporter metrics and receives structured logs. // Implemented by jumpstarter-telemetry; not part of the controller. type TelemetryServiceClient interface { + // Persistent bidirectional stream: telemetry sends scrape requests, + // exporter responds with full metric snapshots (OpenMetrics text). + MetricsStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[MetricsStreamRequest, MetricsStreamResponse], error) // Push structured log entries to the telemetry service for Loki ingest. PushLogs(ctx context.Context, in *PushLogsRequest, opts ...grpc.CallOption) (*PushLogsResponse, error) } @@ -43,6 +47,19 @@ func NewTelemetryServiceClient(cc grpc.ClientConnInterface) TelemetryServiceClie return &telemetryServiceClient{cc} } +func (c *telemetryServiceClient) MetricsStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[MetricsStreamRequest, MetricsStreamResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TelemetryService_ServiceDesc.Streams[0], TelemetryService_MetricsStream_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[MetricsStreamRequest, MetricsStreamResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TelemetryService_MetricsStreamClient = grpc.BidiStreamingClient[MetricsStreamRequest, MetricsStreamResponse] + func (c *telemetryServiceClient) PushLogs(ctx context.Context, in *PushLogsRequest, opts ...grpc.CallOption) (*PushLogsResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PushLogsResponse) @@ -57,9 +74,12 @@ func (c *telemetryServiceClient) PushLogs(ctx context.Context, in *PushLogsReque // All implementations must embed UnimplementedTelemetryServiceServer // for forward compatibility. // -// A service that receives structured logs from exporters and clients. +// A service that reverse-scrapes exporter metrics and receives structured logs. // Implemented by jumpstarter-telemetry; not part of the controller. type TelemetryServiceServer interface { + // Persistent bidirectional stream: telemetry sends scrape requests, + // exporter responds with full metric snapshots (OpenMetrics text). + MetricsStream(grpc.BidiStreamingServer[MetricsStreamRequest, MetricsStreamResponse]) error // Push structured log entries to the telemetry service for Loki ingest. PushLogs(context.Context, *PushLogsRequest) (*PushLogsResponse, error) mustEmbedUnimplementedTelemetryServiceServer() @@ -72,6 +92,9 @@ type TelemetryServiceServer interface { // pointer dereference when methods are called. type UnimplementedTelemetryServiceServer struct{} +func (UnimplementedTelemetryServiceServer) MetricsStream(grpc.BidiStreamingServer[MetricsStreamRequest, MetricsStreamResponse]) error { + return status.Error(codes.Unimplemented, "method MetricsStream not implemented") +} func (UnimplementedTelemetryServiceServer) PushLogs(context.Context, *PushLogsRequest) (*PushLogsResponse, error) { return nil, status.Error(codes.Unimplemented, "method PushLogs not implemented") } @@ -96,6 +119,13 @@ func RegisterTelemetryServiceServer(s grpc.ServiceRegistrar, srv TelemetryServic s.RegisterService(&TelemetryService_ServiceDesc, srv) } +func _TelemetryService_MetricsStream_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TelemetryServiceServer).MetricsStream(&grpc.GenericServerStream[MetricsStreamRequest, MetricsStreamResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TelemetryService_MetricsStreamServer = grpc.BidiStreamingServer[MetricsStreamRequest, MetricsStreamResponse] + func _TelemetryService_PushLogs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PushLogsRequest) if err := dec(in); err != nil { @@ -126,6 +156,13 @@ var TelemetryService_ServiceDesc = grpc.ServiceDesc{ Handler: _TelemetryService_PushLogs_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "MetricsStream", + Handler: _TelemetryService_MetricsStream_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, Metadata: "jumpstarter/v1/telemetry.proto", } diff --git a/controller/internal/service/metrics_merge.go b/controller/internal/service/metrics_merge.go new file mode 100644 index 000000000..378e54af3 --- /dev/null +++ b/controller/internal/service/metrics_merge.go @@ -0,0 +1,239 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "errors" + "io" + "sort" + "strings" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +const ( + labelExporter = "exporter" + labelDriverType = "driver_type" + driverTypeOther = "other" + + // scrapeTimeoutsMetric is incremented once per exporter that does not + // answer a MetricsStream scrape within scrapeTimeout (JEP-0013). + scrapeTimeoutsMetric = "jumpstarter_scrape_timeouts_total" +) + +// DefaultDriverTypeEnum is the JEP-0013 default allowlist for driver_type. +var DefaultDriverTypeEnum = []string{ + "power", "storage", "network", "serial", "console", "video", "composite", +} + +// DefaultExemplarKeys is the JEP-0013 default exemplar allowlist. +var DefaultExemplarKeys = []string{"client", "lease_id"} + +type mergeConfig struct { + exporterName string + driverTypes map[string]struct{} + exemplarKeys map[string]struct{} +} + +type exporterSnapshot struct { + name string + text []byte +} + +func setToMap(values []string) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, v := range values { + if v == "" { + continue + } + out[v] = struct{}{} + } + return out +} + +func (s *TelemetryService) mergeConfigFor(name string) mergeConfig { + types := s.DriverTypeEnum + if len(types) == 0 { + types = DefaultDriverTypeEnum + } + keys := s.ExemplarKeys + if len(keys) == 0 { + keys = DefaultExemplarKeys + } + return mergeConfig{ + exporterName: name, + driverTypes: setToMap(types), + exemplarKeys: setToMap(keys), + } +} + +func parseMetricFamilies(text []byte) ([]*dto.MetricFamily, error) { + dec := expfmt.NewDecoder(bytes.NewReader(text), expfmt.NewFormat(expfmt.TypeOpenMetrics)) + var out []*dto.MetricFamily + for { + mf := new(dto.MetricFamily) + err := dec.Decode(mf) + if errors.Is(err, io.EOF) { + return out, nil + } + if err != nil { + return nil, err + } + out = append(out, mf) + } +} + +func encodeMetricFamilies(w io.Writer, families []*dto.MetricFamily) error { + sorted := append([]*dto.MetricFamily(nil), families...) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].GetName() < sorted[j].GetName() + }) + enc := expfmt.NewEncoder(w, expfmt.NewFormat(expfmt.TypeOpenMetrics)) + for _, f := range sorted { + if f == nil { + continue + } + if err := enc.Encode(f); err != nil { + return err + } + } + if closer, ok := enc.(expfmt.Closer); ok { + return closer.Close() + } + return nil +} + +func mergeSnapshots(snapshots []exporterSnapshot, extra []*dto.MetricFamily, cfgFor func(string) mergeConfig) []*dto.MetricFamily { + byName := map[string]*dto.MetricFamily{} + for _, f := range extra { + if f == nil || f.GetName() == "" { + continue + } + byName[f.GetName()] = f + } + for _, snap := range snapshots { + if len(snap.text) == 0 { + continue + } + families, err := parseMetricFamilies(snap.text) + if err != nil { + continue + } + cfg := cfgFor(snap.name) + for _, f := range families { + applyFamily(f, cfg) + existing, ok := byName[f.GetName()] + if !ok { + byName[f.GetName()] = f + continue + } + existing.Metric = append(existing.Metric, f.Metric...) + } + } + out := make([]*dto.MetricFamily, 0, len(byName)) + for _, f := range byName { + out = append(out, f) + } + return out +} + +func applyFamily(f *dto.MetricFamily, cfg mergeConfig) { + for _, m := range f.GetMetric() { + applyMetric(m, cfg) + } +} + +func applyMetric(m *dto.Metric, cfg mergeConfig) { + setLabel(m, labelExporter, cfg.exporterName) + if v, ok := getLabel(m, labelDriverType); ok { + setLabel(m, labelDriverType, remapDriverType(v, cfg.driverTypes)) + } + if m.Counter != nil { + m.Counter.Exemplar = filterExemplar(m.Counter.Exemplar, cfg.exemplarKeys) + } + if m.Histogram != nil { + for _, b := range m.Histogram.Bucket { + if b != nil { + b.Exemplar = filterExemplar(b.Exemplar, cfg.exemplarKeys) + } + } + if len(m.Histogram.Exemplars) > 0 { + filtered := make([]*dto.Exemplar, 0, len(m.Histogram.Exemplars)) + for _, ex := range m.Histogram.Exemplars { + if fe := filterExemplar(ex, cfg.exemplarKeys); fe != nil { + filtered = append(filtered, fe) + } + } + m.Histogram.Exemplars = filtered + } + } +} + +func remapDriverType(value string, allow map[string]struct{}) string { + if _, ok := allow[value]; ok { + return value + } + return driverTypeOther +} + +func getLabel(m *dto.Metric, name string) (string, bool) { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + return lp.GetValue(), true + } + } + return "", false +} + +func setLabel(m *dto.Metric, name, value string) { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + v := value + lp.Value = &v + return + } + } + m.Label = append(m.Label, labelPair(name, value)) +} + +func labelPair(name, value string) *dto.LabelPair { + n, v := name, value + return &dto.LabelPair{Name: &n, Value: &v} +} + +func filterExemplar(ex *dto.Exemplar, allow map[string]struct{}) *dto.Exemplar { + if ex == nil { + return nil + } + kept := make([]*dto.LabelPair, 0, len(ex.GetLabel())) + for _, lp := range ex.GetLabel() { + if _, ok := allow[lp.GetName()]; !ok { + continue + } + if strings.TrimSpace(lp.GetValue()) == "" { + continue + } + kept = append(kept, lp) + } + if len(kept) == 0 { + return nil + } + ex.Label = kept + return ex +} diff --git a/controller/internal/service/metrics_merge_test.go b/controller/internal/service/metrics_merge_test.go new file mode 100644 index 000000000..5f6dfe70a --- /dev/null +++ b/controller/internal/service/metrics_merge_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +func TestApplyMetric_OverwritesExporterRemapsDriverTypeAndFiltersExemplars(t *testing.T) { + one := 1.0 + m := &dto.Metric{ + Label: []*dto.LabelPair{ + labelPair("exporter", "spoofed"), + labelPair("driver_type", "tuya"), + labelPair("operation", "on"), + }, + Counter: &dto.Counter{ + Value: &one, + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + labelPair("client", "ci"), + labelPair("lease_id", "abc"), + labelPair("trace_id", "drop-me"), + }, + }, + }, + } + applyMetric(m, mergeConfig{ + exporterName: "sidekick", + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + }) + + if got, _ := getLabel(m, "exporter"); got != "sidekick" { + t.Errorf("exporter label = %q, want sidekick", got) + } + if got, _ := getLabel(m, "driver_type"); got != "other" { + t.Errorf("driver_type = %q, want other", got) + } + ex := m.Counter.GetExemplar() + if ex == nil { + t.Fatal("expected exemplar to be kept") + } + got := map[string]string{} + for _, lp := range ex.GetLabel() { + got[lp.GetName()] = lp.GetValue() + } + if got["client"] != "ci" || got["lease_id"] != "abc" { + t.Errorf("exemplar labels = %v, want client=ci lease_id=abc", got) + } + if _, ok := got["trace_id"]; ok { + t.Errorf("trace_id should have been dropped, got %v", got) + } +} + +func TestApplyMetric_KeepsAllowlistedDriverType(t *testing.T) { + m := &dto.Metric{ + Label: []*dto.LabelPair{ + labelPair("driver_type", "power"), + }, + } + applyMetric(m, mergeConfig{ + exporterName: "exp-a", + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + }) + if got, _ := getLabel(m, "driver_type"); got != "power" { + t.Errorf("driver_type = %q, want power", got) + } + if got, _ := getLabel(m, "exporter"); got != "exp-a" { + t.Errorf("exporter = %q, want exp-a", got) + } +} + +func TestMergeSnapshots_CombinesExportersAndSkipsInvalidText(t *testing.T) { + cfg := func(name string) mergeConfig { + return mergeConfig{ + exporterName: name, + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + } + } + textA := []byte(`# TYPE jumpstarter_operations_total counter +# HELP jumpstarter_operations_total Total operations performed. +jumpstarter_operations_total{exporter="a",operation="on",result="success",driver_type="power"} 2.0 +# EOF +`) + textB := []byte(`# TYPE jumpstarter_operations_total counter +jumpstarter_operations_total{exporter="b",operation="off",result="success",driver_type="power"} 3.0 +# EOF +`) + families := mergeSnapshots([]exporterSnapshot{ + {name: "exp-a", text: textA}, + {name: "exp-b", text: textB}, + {name: "bad", text: []byte("not metrics")}, + }, nil, cfg) + + var ops *dto.MetricFamily + for _, f := range families { + if f.GetName() == "jumpstarter_operations_total" { + ops = f + break + } + } + if ops == nil { + t.Fatal("missing jumpstarter_operations_total") + } + if len(ops.Metric) != 2 { + t.Fatalf("got %d series, want 2 (invalid snapshot omitted)", len(ops.Metric)) + } + exporters := map[string]bool{} + for _, m := range ops.Metric { + name, _ := getLabel(m, "exporter") + exporters[name] = true + } + if !exporters["exp-a"] || !exporters["exp-b"] { + t.Errorf("exporters = %v, want exp-a and exp-b", exporters) + } +} + +func TestEncodeMetricFamilies_WritesOpenMetricsEOF(t *testing.T) { + name := "jumpstarter_scrape_timeouts_total" + help := "Exporter MetricsStream scrapes that exceeded scrapeTimeout." + metricType := dto.MetricType_COUNTER + zero := 0.0 + families := []*dto.MetricFamily{{ + Name: &name, + Help: &help, + Type: &metricType, + Metric: []*dto.Metric{{ + Counter: &dto.Counter{Value: &zero}, + }}, + }} + var buf bytes.Buffer + if err := encodeMetricFamilies(&buf, families); err != nil { + t.Fatalf("encode: %v", err) + } + body := buf.String() + if !strings.Contains(body, scrapeTimeoutsMetric) { + t.Errorf("encoded body missing %s:\n%s", scrapeTimeoutsMetric, body) + } + if !strings.Contains(body, "# EOF") { + t.Errorf("OpenMetrics body missing # EOF:\n%s", body) + } + dec := expfmt.NewDecoder(strings.NewReader(body), expfmt.NewFormat(expfmt.TypeOpenMetrics)) + mf := new(dto.MetricFamily) + if err := dec.Decode(mf); err != nil { + t.Fatalf("re-decode: %v", err) + } + if mf.GetName() != scrapeTimeoutsMetric { + t.Errorf("decoded name = %q", mf.GetName()) + } +} diff --git a/controller/internal/service/metrics_stream.go b/controller/internal/service/metrics_stream.go new file mode 100644 index 000000000..b16bb78ef --- /dev/null +++ b/controller/internal/service/metrics_stream.go @@ -0,0 +1,241 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "errors" + "sync" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +var errScrapeTimeout = errors.New("metrics scrape timeout") + +const defaultScrapeTimeout = 7 * time.Second + +// metricsConn is one authenticated MetricsStream. Scrapes are serialized +// because MetricsScrapeRequest has no scrape id (JEP-0013). +type metricsConn struct { + id exporterIdentity + scrapeMu sync.Mutex + mu sync.Mutex + send func(*pb.MetricsStreamResponse) error + // pending is non-nil only while a scrape is waiting for a response. + pending chan *pb.MetricsScrapeResponse + done <-chan struct{} + // started is closed when the Recv loop is running. + started chan struct{} +} + +func (s *TelemetryService) scrapeTimeoutDuration() time.Duration { + if s.ScrapeTimeout > 0 { + return s.ScrapeTimeout + } + return defaultScrapeTimeout +} + +func (s *TelemetryService) initMetricsState() { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.conns == nil { + s.conns = make(map[string]*metricsConn) + } +} + +func (s *TelemetryService) registerConn(c *metricsConn) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.conns == nil { + s.conns = make(map[string]*metricsConn) + } + s.conns[c.id.key()] = c +} + +func (s *TelemetryService) unregisterConn(c *metricsConn) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.conns == nil { + return + } + if cur, ok := s.conns[c.id.key()]; ok && cur == c { + delete(s.conns, c.id.key()) + } +} + +func (s *TelemetryService) snapshotConns() []*metricsConn { + s.stateMu.Lock() + defer s.stateMu.Unlock() + out := make([]*metricsConn, 0, len(s.conns)) + for _, c := range s.conns { + out = append(out, c) + } + return out +} + +// MetricsStream reverse-scrapes one exporter. The first message must be +// MetricsRegister whose identity matches the bearer token. +func (s *TelemetryService) MetricsStream(stream pb.TelemetryService_MetricsStreamServer) error { + s.initMetricsState() + id, err := s.authenticateExporter(stream.Context()) + if err != nil { + return err + } + + first, err := stream.Recv() + if err != nil { + return err + } + reg := first.GetRegister() + if reg == nil { + return status.Error(codes.InvalidArgument, "first MetricsStream message must be MetricsRegister") + } + if reg.GetIdentity() != id.name { + return status.Errorf(codes.PermissionDenied, "register identity %q does not match authenticated exporter %q", reg.GetIdentity(), id.name) + } + + done := make(chan struct{}) + conn := &metricsConn{ + id: id, + send: stream.Send, + done: done, + started: make(chan struct{}), + } + s.registerConn(conn) + defer s.unregisterConn(conn) + + logger := log.FromContext(stream.Context()).WithName("telemetry").WithValues( + "exporter", id.name, + "namespace", id.namespace, + ) + logger.Info("MetricsStream registered") + + var closeDone sync.Once + finish := func() { closeDone.Do(func() { close(done) }) } + defer finish() + close(conn.started) + + for { + msg, err := stream.Recv() + if err != nil { + return err + } + resp := msg.GetScrapeResponse() + if resp == nil { + return status.Error(codes.InvalidArgument, "expected MetricsScrapeResponse after register") + } + conn.mu.Lock() + pending := conn.pending + conn.pending = nil + conn.mu.Unlock() + if pending == nil { + continue + } + select { + case pending <- resp: + case <-done: + return nil + case <-stream.Context().Done(): + return stream.Context().Err() + } + } +} + +func (c *metricsConn) scrape(ctx context.Context, timeout time.Duration) ([]byte, error) { + c.scrapeMu.Lock() + defer c.scrapeMu.Unlock() + + reply := make(chan *pb.MetricsScrapeResponse, 1) + c.mu.Lock() + c.pending = reply + c.mu.Unlock() + defer func() { + c.mu.Lock() + if c.pending == reply { + c.pending = nil + } + c.mu.Unlock() + }() + + err := c.send(&pb.MetricsStreamResponse{ + Msg: &pb.MetricsStreamResponse_ScrapeRequest{ + ScrapeRequest: &pb.MetricsScrapeRequest{}, + }, + }) + if err != nil { + return nil, err + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case resp := <-reply: + if resp == nil { + return nil, errScrapeTimeout + } + return resp.GetMetricsText(), nil + case <-timer.C: + return nil, errScrapeTimeout + case <-ctx.Done(): + return nil, ctx.Err() + case <-c.done: + return nil, context.Canceled + } +} + +func (s *TelemetryService) fanoutScrapes(ctx context.Context) []exporterSnapshot { + s.initMetricsState() + s.initScrapeTimeouts() + timeout := s.scrapeTimeoutDuration() + conns := s.snapshotConns() + if len(conns) == 0 { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var mu sync.Mutex + var snaps []exporterSnapshot + var wg sync.WaitGroup + for _, c := range conns { + wg.Add(1) + go func(c *metricsConn) { + defer wg.Done() + text, err := c.scrape(ctx, timeout) + if err != nil { + if errors.Is(err, errScrapeTimeout) || errors.Is(err, context.DeadlineExceeded) { + s.scrapeTimeouts.Inc() + } + log.FromContext(ctx).WithName("telemetry").V(1).Info("exporter scrape omitted", + "exporter", c.id.name, + "error", err.Error(), + ) + return + } + mu.Lock() + snaps = append(snaps, exporterSnapshot{name: c.id.name, text: text}) + mu.Unlock() + }(c) + } + wg.Wait() + return snaps +} diff --git a/controller/internal/service/metrics_stream_test.go b/controller/internal/service/metrics_stream_test.go new file mode 100644 index 000000000..6f4a50cd0 --- /dev/null +++ b/controller/internal/service/metrics_stream_test.go @@ -0,0 +1,402 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" +) + +const bufconnSize = 1024 * 1024 + +func startTestHub(t *testing.T, timeout time.Duration) (*TelemetryService, pb.TelemetryServiceClient, string) { + t.Helper() + signer := testSigner(t) + svc := &TelemetryService{ + Signer: signer, + MetricsBindAddr: "127.0.0.1:0", + ScrapeTimeout: timeout, + DriverTypeEnum: DefaultDriverTypeEnum, + ExemplarKeys: DefaultExemplarKeys, + } + svc.grpcReady.Store(true) + + shutdown, err := svc.startMetricsHTTP() + if err != nil { + t.Fatalf("startMetricsHTTP: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = shutdown(ctx) + }) + + lis := bufconn.Listen(bufconnSize) + gs := grpc.NewServer() + pb.RegisterTelemetryServiceServer(gs, svc) + go func() { _ = gs.Serve(lis) }() + t.Cleanup(func() { + gs.Stop() + _ = lis.Close() + }) + + conn, err := grpc.NewClient("passthrough:///bufnet", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return lis.DialContext(ctx) + }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Fatalf("grpc.NewClient: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + + return svc, pb.NewTelemetryServiceClient(conn), svc.metricsAddr +} + +func exporterStreamCtx(t *testing.T, svc *TelemetryService, subject string) context.Context { + t.Helper() + token, err := svc.Signer.Token(subject) + if err != nil { + t.Fatalf("Token: %v", err) + } + return metadata.NewOutgoingContext( + context.Background(), + metadata.Pairs("authorization", "Bearer "+token), + ) +} + +func waitRegistered(t *testing.T, svc *TelemetryService, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + conns := svc.snapshotConns() + if len(conns) != n { + time.Sleep(10 * time.Millisecond) + continue + } + ready := true + for _, c := range conns { + if c.started == nil { + ready = false + break + } + select { + case <-c.started: + default: + ready = false + } + if !ready { + break + } + } + if ready { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %d MetricsStream connections, have %d", n, len(svc.snapshotConns())) +} + +func httpGet(t *testing.T, url string) (int, string) { + t.Helper() + client := &http.Client{Timeout: 3 * time.Second} + var resp *http.Response + var lastErr error + for i := 0; i < 30; i++ { + resp, lastErr = client.Get(url) + if lastErr == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if lastErr != nil { + t.Fatalf("GET %s: %v", url, lastErr) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + return resp.StatusCode, string(body) +} + +func TestHealthzAndReadyz(t *testing.T) { + _, _, addr := startTestHub(t, 200*time.Millisecond) + code, body := httpGet(t, "http://"+addr+"/healthz") + if code != http.StatusOK || !strings.Contains(body, "ok") { + t.Fatalf("healthz status=%d body=%q", code, body) + } + code, body = httpGet(t, "http://"+addr+"/readyz") + if code != http.StatusOK || !strings.Contains(body, "ok") { + t.Fatalf("readyz status=%d body=%q", code, body) + } +} + +func TestReadyzNotReadyWhenGRPCDown(t *testing.T) { + svc := &TelemetryService{MetricsBindAddr: "127.0.0.1:0"} + shutdown, err := svc.startMetricsHTTP() + if err != nil { + t.Fatalf("startMetricsHTTP: %v", err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = shutdown(ctx) + }) + code, _ := httpGet(t, "http://"+svc.metricsAddr+"/readyz") + if code != http.StatusServiceUnavailable { + t.Fatalf("readyz status=%d, want 503", code) + } +} + +func TestMetricsStream_RejectsIdentityMismatch(t *testing.T) { + svc, client, _ := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:alice:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "bob"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + _, err = stream.Recv() + if err == nil { + t.Fatal("expected PermissionDenied, got nil") + } + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("code = %v, want PermissionDenied (%v)", status.Code(err), err) + } +} + +func TestMetricsStream_RejectsNonRegisterFirstMessage(t *testing.T) { + svc, client, _ := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:alice:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_ScrapeResponse{ + ScrapeResponse: &pb.MetricsScrapeResponse{MetricsText: []byte("# EOF\n")}, + }, + }); err != nil { + t.Fatalf("Send: %v", err) + } + _, err = stream.Recv() + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("code = %v, want InvalidArgument (%v)", status.Code(err), err) + } +} + +func TestMetricsStream_RejectsNonExporterToken(t *testing.T) { + svc, client, _ := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "client:jumpstarter:ci:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "ci"}, + }, + }); err != nil { + t.Fatalf("Send: %v", err) + } + _, err = stream.Recv() + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("code = %v, want PermissionDenied (%v)", status.Code(err), err) + } +} + +func TestMetricsConn_ScrapeTimeout(t *testing.T) { + done := make(chan struct{}) + c := &metricsConn{ + send: func(*pb.MetricsStreamResponse) error { return nil }, + done: done, + } + _, err := c.scrape(context.Background(), 40*time.Millisecond) + if !errors.Is(err, errScrapeTimeout) { + t.Fatalf("err = %v, want errScrapeTimeout", err) + } +} + +func TestFanout_TimeoutIncrementsCounterWithoutGRPC(t *testing.T) { + svc := &TelemetryService{ScrapeTimeout: 40 * time.Millisecond} + svc.initScrapeTimeouts() + done := make(chan struct{}) + c := &metricsConn{ + id: exporterIdentity{namespace: "jumpstarter", name: "slow"}, + send: func(*pb.MetricsStreamResponse) error { return nil }, + done: done, + } + svc.registerConn(c) + snaps := svc.fanoutScrapes(context.Background()) + if len(snaps) != 0 { + t.Fatalf("timed-out scrape must be omitted, got %d snapshots", len(snaps)) + } + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + var value float64 + var found bool + for _, mf := range mfs { + if mf.GetName() == scrapeTimeoutsMetric || mf.GetName()+"_total" == scrapeTimeoutsMetric { + found = true + if len(mf.Metric) > 0 { + value = mf.Metric[0].GetCounter().GetValue() + } + } + } + if !found || value < 1 { + t.Fatalf("scrape timeout counter = %v found=%v, want >= 1", value, found) + } +} + +func TestFanout_TimeoutOmitsExporterAndIncrementsCounter(t *testing.T) { + svc, client, addr := startTestHub(t, 150*time.Millisecond) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:slow:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "slow"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + // Drain scrape requests so Send on the server does not block, but never reply. + go func() { + for { + if _, err := stream.Recv(); err != nil { + return + } + } + }() + waitRegistered(t, svc, 1) + + code, body := httpGet(t, "http://"+addr+"/metrics") + if code != http.StatusOK { + t.Fatalf("GET /metrics status=%d body=%s", code, body) + } + if strings.Contains(body, "jumpstarter_operations_total") { + t.Errorf("timed-out exporter metrics must be omitted, body:\n%s", body) + } + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + var value float64 + var found bool + for _, mf := range mfs { + if mf.GetName() == scrapeTimeoutsMetric || mf.GetName()+"_total" == scrapeTimeoutsMetric { + found = true + if len(mf.Metric) > 0 { + value = mf.Metric[0].GetCounter().GetValue() + } + } + } + if !found || value < 1 { + t.Fatalf("scrape timeout counter = %v found=%v body:\n%s", value, found, body) + } +} + +func TestFanout_MergesOpenMetricsFromConnectedExporter(t *testing.T) { + svc, client, addr := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:sidekick:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "sidekick"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + + snapshot := []byte(`# TYPE jumpstarter_operations_total counter +# HELP jumpstarter_operations_total Total operations performed. +jumpstarter_operations_total{exporter="spoofed",operation="on",result="success",driver_type="tuya"} 4.0 +# EOF +`) + errCh := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + errCh <- err + return + } + if msg.GetScrapeRequest() == nil { + continue + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_ScrapeResponse{ + ScrapeResponse: &pb.MetricsScrapeResponse{MetricsText: snapshot}, + }, + }); err != nil { + errCh <- err + return + } + } + }() + waitRegistered(t, svc, 1) + + code, body := httpGet(t, "http://"+addr+"/metrics") + if code != http.StatusOK { + t.Fatalf("GET /metrics status=%d body=%s", code, body) + } + if !strings.Contains(body, `exporter="sidekick"`) { + t.Errorf("authenticated exporter label missing:\n%s", body) + } + if strings.Contains(body, `exporter="spoofed"`) { + t.Errorf("spoofed exporter label must be overwritten:\n%s", body) + } + if !strings.Contains(body, `driver_type="other"`) { + t.Errorf("unknown driver_type must remap to other:\n%s", body) + } + select { + case err := <-errCh: + if err != nil && err != io.EOF && status.Code(err) != codes.Canceled && status.Code(err) != codes.Unavailable { + t.Fatalf("stream goroutine: %v", err) + } + default: + } +} diff --git a/controller/internal/service/telemetry_http.go b/controller/internal/service/telemetry_http.go new file mode 100644 index 000000000..6d12daff4 --- /dev/null +++ b/controller/internal/service/telemetry_http.go @@ -0,0 +1,135 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + ctrl "sigs.k8s.io/controller-runtime" +) + +func (s *TelemetryService) initScrapeTimeouts() { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if s.metricsRegistry != nil { + return + } + s.metricsRegistry = prometheus.NewRegistry() + s.scrapeTimeouts = prometheus.NewCounter(prometheus.CounterOpts{ + Name: scrapeTimeoutsMetric, + Help: "Exporter MetricsStream scrapes that exceeded scrapeTimeout.", + }) + s.metricsRegistry.MustRegister(s.scrapeTimeouts) +} + +func metricsHTTPEnabled(addr string) bool { + return addr != "" && addr != "0" +} + +func (s *TelemetryService) startMetricsHTTP() (func(context.Context) error, error) { + addr := s.MetricsBindAddr + if !metricsHTTPEnabled(addr) { + return nil, nil + } + s.initMetricsState() + s.initScrapeTimeouts() + + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + s.metricsAddr = ln.Addr().String() + + mux := http.NewServeMux() + mux.HandleFunc("/metrics", s.handleMetrics) + mux.HandleFunc("/healthz", s.handleHealthz) + mux.HandleFunc("/readyz", s.handleReadyz) + + writeTimeout := s.scrapeTimeoutDuration() + 15*time.Second + if writeTimeout < 30*time.Second { + writeTimeout = 30 * time.Second + } + + srv := &http.Server{ + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: writeTimeout, + IdleTimeout: 5 * time.Minute, + } + go func() { + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { + ctrl.Log.WithName("telemetry").Error(err, "metrics HTTP server stopped unexpectedly") + } + }() + ctrl.Log.WithName("telemetry").Info("Telemetry metrics HTTP listening", + "addr", s.metricsAddr, + ) + return srv.Shutdown, nil +} + +func (s *TelemetryService) handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + +// handleReadyz is 200 once the gRPC listener is bound. +// JEP DD-7 also gates on Loki reachability and a connected exporter; Loki is +// Phase 3 PR D, and requiring an exporter would keep an empty lab unready +// (Prometheus could not scrape jumpstarter_scrape_timeouts_total). +func (s *TelemetryService) handleReadyz(w http.ResponseWriter, _ *http.Request) { + if !s.grpcReady.Load() { + http.Error(w, "grpc not ready\n", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + +func (s *TelemetryService) handleMetrics(w http.ResponseWriter, r *http.Request) { + s.initScrapeTimeouts() + snaps := s.fanoutScrapes(r.Context()) + + var extra []*dto.MetricFamily + if s.metricsRegistry != nil { + gathered, err := s.metricsRegistry.Gather() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + extra = gathered + } + + families := mergeSnapshots(snaps, extra, s.mergeConfigFor) + + var buf bytes.Buffer + if err := encodeMetricFamilies(&buf, families); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", string(expfmt.NewFormat(expfmt.TypeOpenMetrics))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(buf.Bytes()) +} diff --git a/controller/internal/service/telemetry_identity.go b/controller/internal/service/telemetry_identity.go new file mode 100644 index 000000000..e1f23de4e --- /dev/null +++ b/controller/internal/service/telemetry_identity.go @@ -0,0 +1,62 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "strings" + + "github.com/jumpstarter-dev/jumpstarter/controller/internal/authentication" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// exporterIdentity is the exporter CRD namespace/name claimed by a bearer token. +type exporterIdentity struct { + namespace string + name string +} + +func (id exporterIdentity) key() string { + return id.namespace + "/" + id.name +} + +func parseExporterSubject(subject string) (exporterIdentity, error) { + parts := strings.SplitN(subject, ":", 4) + if len(parts) != 4 || parts[0] != "exporter" { + return exporterIdentity{}, status.Errorf(codes.PermissionDenied, "token is not an exporter token") + } + if parts[1] == "" || parts[2] == "" { + return exporterIdentity{}, status.Errorf(codes.PermissionDenied, "token has incomplete exporter identity") + } + return exporterIdentity{namespace: parts[1], name: parts[2]}, nil +} + +func (s *TelemetryService) authenticateExporter(ctx context.Context) (exporterIdentity, error) { + token, err := authentication.BearerTokenFromContext(ctx) + if err != nil { + return exporterIdentity{}, err + } + if s.Signer == nil { + return exporterIdentity{}, status.Error(codes.Internal, "telemetry signer is not configured") + } + subject, err := s.Signer.ParseSubject(token) + if err != nil { + return exporterIdentity{}, status.Errorf(codes.Unauthenticated, "invalid token: %v", err) + } + return parseExporterSubject(subject) +} diff --git a/controller/internal/service/telemetry_service.go b/controller/internal/service/telemetry_service.go index 80844b807..e3c60bb77 100644 --- a/controller/internal/service/telemetry_service.go +++ b/controller/internal/service/telemetry_service.go @@ -22,17 +22,17 @@ import ( "fmt" "net" "strings" + "sync" + "sync/atomic" "time" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" - "github.com/jumpstarter-dev/jumpstarter/controller/internal/authentication" "github.com/jumpstarter-dev/jumpstarter/controller/internal/oidc" pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" - "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/reflection" - "google.golang.org/grpc/status" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -69,42 +69,47 @@ var reservedExtraFieldKeys = map[string]struct{}{ type TelemetryService struct { pb.UnimplementedTelemetryServiceServer - // BindAddr is the TCP address to listen on (e.g. ":9093"). + // BindAddr is the TCP address to listen on for gRPC (e.g. ":9093"). BindAddr string - // Signer is used to validate bearer tokens on every PushLogs call. - // Tokens are issued by the controller from the same CONTROLLER_KEY seed, - // so the telemetry binary can verify them locally without a k8s client. + // MetricsBindAddr is the TCP address for HTTP GET /metrics, /healthz, and + // /readyz. Empty or "0" disables the HTTP server (tests). Production + // default is :8080 — a dedicated port, not multiplexed onto gRPC :9093. + MetricsBindAddr string + + // ScrapeTimeout is the fan-out wait for MetricsStream responses (JEP default 7s). + ScrapeTimeout time.Duration + + // DriverTypeEnum allowlist; unknown driver_type values are remapped to "other". + DriverTypeEnum []string + + // ExemplarKeys allowlist applied on the telemetry merge path. + ExemplarKeys []string + + // Signer is used to validate bearer tokens on every PushLogs call and + // MetricsStream. Tokens are issued by the controller from the same + // CONTROLLER_KEY seed, so the telemetry binary can verify them locally + // without a k8s client. Signer *oidc.Signer + + stateMu sync.Mutex + conns map[string]*metricsConn + scrapeTimeouts prometheus.Counter + metricsRegistry *prometheus.Registry + metricsAddr string + grpcReady atomic.Bool } // PushLogs receives a batch of structured log entries and writes them via the // controller-runtime logger (structured JSON to stdout). // Future phase: forward to Loki push API. func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest) (*pb.PushLogsResponse, error) { - token, err := authentication.BearerTokenFromContext(ctx) + id, err := s.authenticateExporter(ctx) if err != nil { return nil, err } - - // Validate token and extract the subject (format: exporter:namespace:name:uid). - subject, err := s.Signer.ParseSubject(token) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "invalid token: %v", err) - } - - // Only exporter tokens are allowed to push logs. Any other validly-signed - // token (e.g. a client token) is rejected immediately so that the identity - // checks below always have a non-empty claimedName/claimedNamespace. - parts := strings.SplitN(subject, ":", 4) - if len(parts) != 4 || parts[0] != "exporter" { - return nil, status.Errorf(codes.PermissionDenied, "token is not an exporter token") - } - claimedNamespace := parts[1] - claimedName := parts[2] - if claimedNamespace == "" || claimedName == "" { - return nil, status.Errorf(codes.PermissionDenied, "token has incomplete exporter identity") - } + claimedNamespace := id.namespace + claimedName := id.name // Use context-based logger so tests can inject their own via logf.IntoContext. logger := log.FromContext(ctx).WithName("telemetry") @@ -260,9 +265,21 @@ func (s *TelemetryService) Start(ctx context.Context) error { return fmt.Errorf("telemetry: listen %s: %w", s.BindAddr, err) } + s.initMetricsState() + s.initScrapeTimeouts() + s.grpcReady.Store(true) + + httpShutdown, err := s.startMetricsHTTP() + if err != nil { + s.grpcReady.Store(false) + _ = lis.Close() + return fmt.Errorf("telemetry: metrics HTTP listen %s: %w", s.MetricsBindAddr, err) + } + srv := grpc.NewServer( grpc.Creds(creds), grpc.ChainUnaryInterceptor(recovery.UnaryServerInterceptor()), + grpc.ChainStreamInterceptor(recovery.StreamServerInterceptor()), ) pb.RegisterTelemetryServiceServer(srv, s) reflection.Register(srv) @@ -276,12 +293,24 @@ func (s *TelemetryService) Start(ctx context.Context) error { select { case <-ctx.Done(): + s.grpcReady.Store(false) + if httpShutdown != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = httpShutdown(shutdownCtx) + } srv.GracefulStop() if err := <-errCh; err != nil && !errors.Is(err, grpc.ErrServerStopped) { return err } return nil case err := <-errCh: + s.grpcReady.Store(false) + if httpShutdown != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = httpShutdown(shutdownCtx) + } srv.Stop() return err } diff --git a/protocol/proto/jumpstarter/v1/telemetry.proto b/protocol/proto/jumpstarter/v1/telemetry.proto index 200748201..fbbfd00ee 100644 --- a/protocol/proto/jumpstarter/v1/telemetry.proto +++ b/protocol/proto/jumpstarter/v1/telemetry.proto @@ -6,13 +6,44 @@ package jumpstarter.v1; import "google/protobuf/timestamp.proto"; -// A service that receives structured logs from exporters and clients. +// A service that reverse-scrapes exporter metrics and receives structured logs. // Implemented by jumpstarter-telemetry; not part of the controller. service TelemetryService { + // Persistent bidirectional stream: telemetry sends scrape requests, + // exporter responds with full metric snapshots (OpenMetrics text). + rpc MetricsStream(stream MetricsStreamRequest) returns (stream MetricsStreamResponse); + // Push structured log entries to the telemetry service for Loki ingest. rpc PushLogs(PushLogsRequest) returns (PushLogsResponse); } +// Exporter → Telemetry +message MetricsStreamRequest { + oneof msg { + MetricsRegister register = 1; // First message: identify this exporter. + MetricsScrapeResponse scrape_response = 2; // Subsequent: reply to a scrape. + } +} + +message MetricsRegister { + string identity = 1; // Exporter CRD name (verified against the auth token by the server). +} + +message MetricsScrapeResponse { + bytes metrics_text = 1; // generate_latest() OpenMetrics output. + google.protobuf.Timestamp timestamp = 2; +} + +// Telemetry → Exporter +message MetricsStreamResponse { + oneof msg { + MetricsScrapeRequest scrape_request = 1; + } +} + +// Empty request: "send your /metrics now". +message MetricsScrapeRequest {} + // Request to push log entries to the telemetry service. message PushLogsRequest { repeated LogEntry entries = 1; // Log entries to push. diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py index 30983ccc3..80e4bc2bc 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.py @@ -25,24 +25,34 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x0fPushLogsRequest\x122\n\x07entries\x18\x01 \x03(\x0b2\x18.jumpstarter.v1.LogEntryR\x07entries"H\n\x10PushLogsResponse\x12\x1a\n\x08accepted\x18\x01 \x01(\rR\x08accepted\x12\x18\n\x07dropped\x18\x02 \x01(\rR\x07dropped"\xe5\x03\n\x08LogEntry\x128\n\ttimestamp\x18\x01 \x01(\x0b2\x1a.google.protobuf.TimestampR\ttimestamp\x12\x1a\n\x08severity\x18\x02 \x01(\tR\x08severity\x12\x18\n\x07message\x18\x03 \x01(\tR\x07message\x12\x1c\n\tcomponent\x18\x04 \x01(\tR\tcomponent\x12\x1a\n\x08exporter\x18\x05 \x01(\tR\x08exporter\x12\x14\n\x05lease\x18\x06 \x01(\tR\x05lease\x12\x16\n\x06client\x18\x07 \x01(\tR\x06client\x12\x1c\n\toperation\x18\x08 \x01(\tR\toperation\x12\x16\n\x06result\x18\t \x01(\tR\x06result\x12\x1f\n\x0bdriver_type\x18\n \x01(\tR\ndriverType\x12L\n\x0cextra_fields\x18\x0b \x03(\x0b2).jumpstarter.v1.LogEntry.ExtraFieldsEntryR\x0bextraFields\x12\x1c\n\tnamespace\x18\x0c \x01(\tR\tnamespace\x1a>\n\x10ExtraFieldsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x028\x012a\n\x10TelemetryService\x12M\n\x08PushLogs\x12\x1f.jumpstarter.v1.PushLogsRequest\x1a .jumpstarter.v1.PushLogsResponseB\xd1\x01\n\x12com.jumpstarter.v1B\x0eTelemetryProtoP\x01ZRgithub.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1\xa2\x02\x03JXX\xaa\x02\x0eJumpstarter.V1\xca\x02\x0eJumpstarter\\V1\xe2\x02\x1aJumpstarter\\V1\\GPBMetadata\xea\x02\x0fJumpstarter::V1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1ejumpstarter/v1/telemetry.proto\x12\x0ejumpstarter.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xae\x01\n\x14MetricsStreamRequest\x12=\n\x08register\x18\x01 \x01(\x0b\x32\x1f.jumpstarter.v1.MetricsRegisterH\x00R\x08register\x12P\n\x0fscrape_response\x18\x02 \x01(\x0b\x32%.jumpstarter.v1.MetricsScrapeResponseH\x00R\x0escrapeResponseB\x05\n\x03msg\"-\n\x0fMetricsRegister\x12\x1a\n\x08identity\x18\x01 \x01(\tR\x08identity\"t\n\x15MetricsScrapeResponse\x12!\n\x0cmetrics_text\x18\x01 \x01(\x0cR\x0bmetricsText\x12\x38\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ttimestamp\"m\n\x15MetricsStreamResponse\x12M\n\x0escrape_request\x18\x01 \x01(\x0b\x32$.jumpstarter.v1.MetricsScrapeRequestH\x00R\rscrapeRequestB\x05\n\x03msg\"\x16\n\x14MetricsScrapeRequest\"E\n\x0fPushLogsRequest\x12\x32\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x18.jumpstarter.v1.LogEntryR\x07\x65ntries\"H\n\x10PushLogsResponse\x12\x1a\n\x08\x61\x63\x63\x65pted\x18\x01 \x01(\rR\x08\x61\x63\x63\x65pted\x12\x18\n\x07\x64ropped\x18\x02 \x01(\rR\x07\x64ropped\"\xe5\x03\n\x08LogEntry\x12\x38\n\ttimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ttimestamp\x12\x1a\n\x08severity\x18\x02 \x01(\tR\x08severity\x12\x18\n\x07message\x18\x03 \x01(\tR\x07message\x12\x1c\n\tcomponent\x18\x04 \x01(\tR\tcomponent\x12\x1a\n\x08\x65xporter\x18\x05 \x01(\tR\x08\x65xporter\x12\x14\n\x05lease\x18\x06 \x01(\tR\x05lease\x12\x16\n\x06\x63lient\x18\x07 \x01(\tR\x06\x63lient\x12\x1c\n\toperation\x18\x08 \x01(\tR\toperation\x12\x16\n\x06result\x18\t \x01(\tR\x06result\x12\x1f\n\x0b\x64river_type\x18\n \x01(\tR\ndriverType\x12L\n\x0c\x65xtra_fields\x18\x0b \x03(\x0b\x32).jumpstarter.v1.LogEntry.ExtraFieldsEntryR\x0b\x65xtraFields\x12\x1c\n\tnamespace\x18\x0c \x01(\tR\tnamespace\x1a>\n\x10\x45xtraFieldsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x32\xc3\x01\n\x10TelemetryService\x12`\n\rMetricsStream\x12$.jumpstarter.v1.MetricsStreamRequest\x1a%.jumpstarter.v1.MetricsStreamResponse(\x01\x30\x01\x12M\n\x08PushLogs\x12\x1f.jumpstarter.v1.PushLogsRequest\x1a .jumpstarter.v1.PushLogsResponseB}\n\x12\x63om.jumpstarter.v1B\x0eTelemetryProtoP\x01\xa2\x02\x03JXX\xaa\x02\x0eJumpstarter.V1\xca\x02\x0eJumpstarter\\V1\xe2\x02\x1aJumpstarter\\V1\\GPBMetadata\xea\x02\x0fJumpstarter::V1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'jumpstarter.v1.telemetry_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\022com.jumpstarter.v1B\016TelemetryProtoP\001ZRgithub.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1\242\002\003JXX\252\002\016Jumpstarter.V1\312\002\016Jumpstarter\\V1\342\002\032Jumpstarter\\V1\\GPBMetadata\352\002\017Jumpstarter::V1' + _globals['DESCRIPTOR']._serialized_options = b'\n\022com.jumpstarter.v1B\016TelemetryProtoP\001\242\002\003JXX\252\002\016Jumpstarter.V1\312\002\016Jumpstarter\\V1\342\002\032Jumpstarter\\V1\\GPBMetadata\352\002\017Jumpstarter::V1' _globals['_LOGENTRY_EXTRAFIELDSENTRY']._loaded_options = None _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_options = b'8\001' - _globals['_PUSHLOGSREQUEST']._serialized_start=83 - _globals['_PUSHLOGSREQUEST']._serialized_end=152 - _globals['_PUSHLOGSRESPONSE']._serialized_start=154 - _globals['_PUSHLOGSRESPONSE']._serialized_end=226 - _globals['_LOGENTRY']._serialized_start=229 - _globals['_LOGENTRY']._serialized_end=714 - _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_start=652 - _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_end=714 - _globals['_TELEMETRYSERVICE']._serialized_start=716 - _globals['_TELEMETRYSERVICE']._serialized_end=813 + _globals['_METRICSSTREAMREQUEST']._serialized_start=84 + _globals['_METRICSSTREAMREQUEST']._serialized_end=258 + _globals['_METRICSREGISTER']._serialized_start=260 + _globals['_METRICSREGISTER']._serialized_end=305 + _globals['_METRICSSCRAPERESPONSE']._serialized_start=307 + _globals['_METRICSSCRAPERESPONSE']._serialized_end=423 + _globals['_METRICSSTREAMRESPONSE']._serialized_start=425 + _globals['_METRICSSTREAMRESPONSE']._serialized_end=534 + _globals['_METRICSSCRAPEREQUEST']._serialized_start=536 + _globals['_METRICSSCRAPEREQUEST']._serialized_end=558 + _globals['_PUSHLOGSREQUEST']._serialized_start=560 + _globals['_PUSHLOGSREQUEST']._serialized_end=629 + _globals['_PUSHLOGSRESPONSE']._serialized_start=631 + _globals['_PUSHLOGSRESPONSE']._serialized_end=703 + _globals['_LOGENTRY']._serialized_start=706 + _globals['_LOGENTRY']._serialized_end=1191 + _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_start=1129 + _globals['_LOGENTRY_EXTRAFIELDSENTRY']._serialized_end=1191 + _globals['_TELEMETRYSERVICE']._serialized_start=1194 + _globals['_TELEMETRYSERVICE']._serialized_end=1389 # @@protoc_insertion_point(module_scope) diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi index 2f28d9f24..968f7ea75 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2.pyi @@ -19,6 +19,103 @@ else: DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +@typing.final +class MetricsStreamRequest(google.protobuf.message.Message): + """Exporter → Telemetry""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REGISTER_FIELD_NUMBER: builtins.int + SCRAPE_RESPONSE_FIELD_NUMBER: builtins.int + @property + def register(self) -> Global___MetricsRegister: + """First message: identify this exporter.""" + + @property + def scrape_response(self) -> Global___MetricsScrapeResponse: + """Subsequent: reply to a scrape.""" + + def __init__( + self, + *, + register: Global___MetricsRegister | None = ..., + scrape_response: Global___MetricsScrapeResponse | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["msg", b"msg", "register", b"register", "scrape_response", b"scrape_response"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["msg", b"msg", "register", b"register", "scrape_response", b"scrape_response"]) -> None: ... + def WhichOneof(self, oneof_group: typing.Literal["msg", b"msg"]) -> typing.Literal["register", "scrape_response"] | None: ... + +Global___MetricsStreamRequest: typing_extensions.TypeAlias = MetricsStreamRequest + +@typing.final +class MetricsRegister(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + IDENTITY_FIELD_NUMBER: builtins.int + identity: builtins.str + """Exporter CRD name (verified against the auth token by the server).""" + def __init__( + self, + *, + identity: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["identity", b"identity"]) -> None: ... + +Global___MetricsRegister: typing_extensions.TypeAlias = MetricsRegister + +@typing.final +class MetricsScrapeResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + METRICS_TEXT_FIELD_NUMBER: builtins.int + TIMESTAMP_FIELD_NUMBER: builtins.int + metrics_text: builtins.bytes + """generate_latest() OpenMetrics output.""" + @property + def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + metrics_text: builtins.bytes = ..., + timestamp: google.protobuf.timestamp_pb2.Timestamp | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["timestamp", b"timestamp"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["metrics_text", b"metrics_text", "timestamp", b"timestamp"]) -> None: ... + +Global___MetricsScrapeResponse: typing_extensions.TypeAlias = MetricsScrapeResponse + +@typing.final +class MetricsStreamResponse(google.protobuf.message.Message): + """Telemetry → Exporter""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SCRAPE_REQUEST_FIELD_NUMBER: builtins.int + @property + def scrape_request(self) -> Global___MetricsScrapeRequest: ... + def __init__( + self, + *, + scrape_request: Global___MetricsScrapeRequest | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["msg", b"msg", "scrape_request", b"scrape_request"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["msg", b"msg", "scrape_request", b"scrape_request"]) -> None: ... + def WhichOneof(self, oneof_group: typing.Literal["msg", b"msg"]) -> typing.Literal["scrape_request"] | None: ... + +Global___MetricsStreamResponse: typing_extensions.TypeAlias = MetricsStreamResponse + +@typing.final +class MetricsScrapeRequest(google.protobuf.message.Message): + """Empty request: "send your /metrics now".""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +Global___MetricsScrapeRequest: typing_extensions.TypeAlias = MetricsScrapeRequest + @typing.final class PushLogsRequest(google.protobuf.message.Message): """Request to push log entries to the telemetry service.""" @@ -96,6 +193,7 @@ class LogEntry(google.protobuf.message.Message): RESULT_FIELD_NUMBER: builtins.int DRIVER_TYPE_FIELD_NUMBER: builtins.int EXTRA_FIELDS_FIELD_NUMBER: builtins.int + NAMESPACE_FIELD_NUMBER: builtins.int severity: builtins.str """Log severity: debug, info, warning, error, critical.""" message: builtins.str @@ -115,7 +213,9 @@ class LogEntry(google.protobuf.message.Message): driver_type: builtins.str """Log body: driver category (storage, power, network, etc.).""" namespace: builtins.str - """Loki stream label: Kubernetes namespace (bounded by cluster size).""" + """Capped at 16 entries, 64-char keys, 256-char values. + Loki stream label: Kubernetes namespace (bounded by cluster size). + """ @property def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: """When the log was emitted.""" diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py index 7be5e24a0..e34c217b6 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.py @@ -6,7 +6,7 @@ class TelemetryServiceStub: - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ @@ -16,6 +16,11 @@ def __init__(self, channel): Args: channel: A grpc.Channel. """ + self.MetricsStream = channel.stream_stream( + '/jumpstarter.v1.TelemetryService/MetricsStream', + request_serializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamRequest.SerializeToString, + response_deserializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamResponse.FromString, + _registered_method=True) self.PushLogs = channel.unary_unary( '/jumpstarter.v1.TelemetryService/PushLogs', request_serializer=jumpstarter_dot_v1_dot_telemetry__pb2.PushLogsRequest.SerializeToString, @@ -24,10 +29,18 @@ def __init__(self, channel): class TelemetryServiceServicer: - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ + def MetricsStream(self, request_iterator, context): + """Persistent bidirectional stream: telemetry sends scrape requests, + exporter responds with full metric snapshots (OpenMetrics text). + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def PushLogs(self, request, context): """Push structured log entries to the telemetry service for Loki ingest. """ @@ -38,6 +51,11 @@ def PushLogs(self, request, context): def add_TelemetryServiceServicer_to_server(servicer, server): rpc_method_handlers = { + 'MetricsStream': grpc.stream_stream_rpc_method_handler( + servicer.MetricsStream, + request_deserializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamRequest.FromString, + response_serializer=jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamResponse.SerializeToString, + ), 'PushLogs': grpc.unary_unary_rpc_method_handler( servicer.PushLogs, request_deserializer=jumpstarter_dot_v1_dot_telemetry__pb2.PushLogsRequest.FromString, @@ -52,10 +70,37 @@ def add_TelemetryServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class TelemetryService: - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ + @staticmethod + def MetricsStream(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream( + request_iterator, + target, + '/jumpstarter.v1.TelemetryService/MetricsStream', + jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamRequest.SerializeToString, + jumpstarter_dot_v1_dot_telemetry__pb2.MetricsStreamResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def PushLogs(request, target, diff --git a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi index dc163b903..d810b4378 100644 --- a/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi +++ b/python/packages/jumpstarter-protocol/jumpstarter_protocol/jumpstarter/v1/telemetry_pb2_grpc.pyi @@ -25,6 +25,22 @@ class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext): # type: GRPC_GENERATED_VERSION: str GRPC_VERSION: str +_TelemetryServiceMetricsStreamType = typing_extensions.TypeVar( + '_TelemetryServiceMetricsStreamType', + grpc.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], + grpc.aio.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], + default=grpc.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], +) + _TelemetryServicePushLogsType = typing_extensions.TypeVar( '_TelemetryServicePushLogsType', grpc.UnaryUnaryMultiCallable[ @@ -41,13 +57,17 @@ _TelemetryServicePushLogsType = typing_extensions.TypeVar( ], ) -class TelemetryServiceStub(typing.Generic[_TelemetryServicePushLogsType]): - """A service that receives structured logs from exporters and clients. +class TelemetryServiceStub(typing.Generic[_TelemetryServiceMetricsStreamType, _TelemetryServicePushLogsType]): + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ @typing.overload def __init__(self: TelemetryServiceStub[ + grpc.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], grpc.UnaryUnaryMultiCallable[ jumpstarter.v1.telemetry_pb2.PushLogsRequest, jumpstarter.v1.telemetry_pb2.PushLogsResponse, @@ -56,16 +76,29 @@ class TelemetryServiceStub(typing.Generic[_TelemetryServicePushLogsType]): @typing.overload def __init__(self: TelemetryServiceStub[ + grpc.aio.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], grpc.aio.UnaryUnaryMultiCallable[ jumpstarter.v1.telemetry_pb2.PushLogsRequest, jumpstarter.v1.telemetry_pb2.PushLogsResponse, ], ], channel: grpc.aio.Channel) -> None: ... + MetricsStream: _TelemetryServiceMetricsStreamType + """Persistent bidirectional stream: telemetry sends scrape requests, + exporter responds with full metric snapshots (OpenMetrics text). + """ + PushLogs: _TelemetryServicePushLogsType """Push structured log entries to the telemetry service for Loki ingest.""" TelemetryServiceAsyncStub: typing_extensions.TypeAlias = TelemetryServiceStub[ + grpc.aio.StreamStreamMultiCallable[ + jumpstarter.v1.telemetry_pb2.MetricsStreamRequest, + jumpstarter.v1.telemetry_pb2.MetricsStreamResponse, + ], grpc.aio.UnaryUnaryMultiCallable[ jumpstarter.v1.telemetry_pb2.PushLogsRequest, jumpstarter.v1.telemetry_pb2.PushLogsResponse, @@ -73,10 +106,20 @@ TelemetryServiceAsyncStub: typing_extensions.TypeAlias = TelemetryServiceStub[ ] class TelemetryServiceServicer(metaclass=abc.ABCMeta): - """A service that receives structured logs from exporters and clients. + """A service that reverse-scrapes exporter metrics and receives structured logs. Implemented by jumpstarter-telemetry; not part of the controller. """ + @abc.abstractmethod + def MetricsStream( + self, + request_iterator: _MaybeAsyncIterator[jumpstarter.v1.telemetry_pb2.MetricsStreamRequest], + context: _ServicerContext, + ) -> typing.Union[collections.abc.Iterator[jumpstarter.v1.telemetry_pb2.MetricsStreamResponse], collections.abc.AsyncIterator[jumpstarter.v1.telemetry_pb2.MetricsStreamResponse]]: + """Persistent bidirectional stream: telemetry sends scrape requests, + exporter responds with full metric snapshots (OpenMetrics text). + """ + @abc.abstractmethod def PushLogs( self, From 2423e947af0d1eb234b164078b6c8fd7dd65d2f6 Mon Sep 17 00:00:00 2001 From: Roddie Kieley Date: Tue, 1 Sep 2026 20:35:14 -0230 Subject: [PATCH 2/5] fix: surface MetricsStream OpenMetrics parse failures (JEP-0013 Phase 3) Stop silently dropping unparseable exporter snapshots. Log the exporter and error, and increment jumpstarter_metrics_parse_errors_total so reverse-scrape omissions are visible on the same /metrics response. Co-authored-by: Cursor --- controller/internal/service/metrics_merge.go | 9 +- .../internal/service/metrics_merge_test.go | 106 ++++++++++++++++-- .../internal/service/metrics_stream_test.go | 67 +++++++++++ controller/internal/service/telemetry_http.go | 22 +++- .../internal/service/telemetry_service.go | 1 + 5 files changed, 190 insertions(+), 15 deletions(-) diff --git a/controller/internal/service/metrics_merge.go b/controller/internal/service/metrics_merge.go index 378e54af3..484b4e27e 100644 --- a/controller/internal/service/metrics_merge.go +++ b/controller/internal/service/metrics_merge.go @@ -35,6 +35,10 @@ const ( // scrapeTimeoutsMetric is incremented once per exporter that does not // answer a MetricsStream scrape within scrapeTimeout (JEP-0013). scrapeTimeoutsMetric = "jumpstarter_scrape_timeouts_total" + + // metricsParseErrorsMetric is incremented once per exporter snapshot that + // MetricsStream delivered but OpenMetrics parse rejected (JEP-0013). + metricsParseErrorsMetric = "jumpstarter_metrics_parse_errors_total" ) // DefaultDriverTypeEnum is the JEP-0013 default allowlist for driver_type. @@ -119,7 +123,7 @@ func encodeMetricFamilies(w io.Writer, families []*dto.MetricFamily) error { return nil } -func mergeSnapshots(snapshots []exporterSnapshot, extra []*dto.MetricFamily, cfgFor func(string) mergeConfig) []*dto.MetricFamily { +func mergeSnapshots(snapshots []exporterSnapshot, extra []*dto.MetricFamily, cfgFor func(string) mergeConfig, onParseError func(exporter string, err error)) []*dto.MetricFamily { byName := map[string]*dto.MetricFamily{} for _, f := range extra { if f == nil || f.GetName() == "" { @@ -133,6 +137,9 @@ func mergeSnapshots(snapshots []exporterSnapshot, extra []*dto.MetricFamily, cfg } families, err := parseMetricFamilies(snap.text) if err != nil { + if onParseError != nil { + onParseError(snap.name, err) + } continue } cfg := cfgFor(snap.name) diff --git a/controller/internal/service/metrics_merge_test.go b/controller/internal/service/metrics_merge_test.go index 5f6dfe70a..72bdbbbc4 100644 --- a/controller/internal/service/metrics_merge_test.go +++ b/controller/internal/service/metrics_merge_test.go @@ -18,6 +18,7 @@ package service import ( "bytes" + "errors" "strings" "testing" @@ -25,6 +26,24 @@ import ( "github.com/prometheus/common/expfmt" ) +func labeledCounterValue(t *testing.T, mfs []*dto.MetricFamily, name, label, want string) float64 { + t.Helper() + for _, mf := range mfs { + if mf.GetName() != name && mf.GetName()+"_total" != name { + continue + } + for _, m := range mf.Metric { + for _, lp := range m.GetLabel() { + if lp.GetName() == label && lp.GetValue() == want { + return m.GetCounter().GetValue() + } + } + } + } + t.Fatalf("%s{%s=%q} missing from gathered families", name, label, want) + return 0 +} + func TestApplyMetric_OverwritesExporterRemapsDriverTypeAndFiltersExemplars(t *testing.T) { one := 1.0 m := &dto.Metric{ @@ -91,14 +110,32 @@ func TestApplyMetric_KeepsAllowlistedDriverType(t *testing.T) { } } -func TestMergeSnapshots_CombinesExportersAndSkipsInvalidText(t *testing.T) { - cfg := func(name string) mergeConfig { - return mergeConfig{ - exporterName: name, - driverTypes: setToMap(DefaultDriverTypeEnum), - exemplarKeys: setToMap(DefaultExemplarKeys), - } +// pythonOpenMetricsWithExemplar is the prometheus_client OpenMetrics form +// that prometheus/common v0.62's text fallback cannot parse (it treats '#' as +// a timestamp). This is the body exporters send after driver operations. +const pythonOpenMetricsWithExemplar = `# TYPE jumpstarter_operation_duration_seconds histogram +jumpstarter_operation_duration_seconds_bucket{exporter="sidekick",le="0.005",operation="on",result="success",driver_type="power"} 1.0 # {lease_id="lease-1"} 0.00262 1788299670.149 +jumpstarter_operation_duration_seconds_sum{exporter="sidekick",operation="on",result="success",driver_type="power"} 0.00262 +jumpstarter_operation_duration_seconds_count{exporter="sidekick",operation="on",result="success",driver_type="power"} 1.0 +# EOF +` + +func testMergeCfg(name string) mergeConfig { + return mergeConfig{ + exporterName: name, + driverTypes: setToMap(DefaultDriverTypeEnum), + exemplarKeys: setToMap(DefaultExemplarKeys), + } +} + +func TestParseMetricFamilies_OpenMetricsExemplarSuffixFails(t *testing.T) { + _, err := parseMetricFamilies([]byte(pythonOpenMetricsWithExemplar)) + if err == nil { + t.Fatal("expected parse error for OpenMetrics exemplar suffix") } +} + +func TestMergeSnapshots_CombinesExportersAndReportsInvalidText(t *testing.T) { textA := []byte(`# TYPE jumpstarter_operations_total counter # HELP jumpstarter_operations_total Total operations performed. jumpstarter_operations_total{exporter="a",operation="on",result="success",driver_type="power"} 2.0 @@ -108,11 +145,18 @@ jumpstarter_operations_total{exporter="a",operation="on",result="success",driver jumpstarter_operations_total{exporter="b",operation="off",result="success",driver_type="power"} 3.0 # EOF `) + type parseErrCall struct { + exporter string + err error + } + var calls []parseErrCall families := mergeSnapshots([]exporterSnapshot{ {name: "exp-a", text: textA}, {name: "exp-b", text: textB}, {name: "bad", text: []byte("not metrics")}, - }, nil, cfg) + }, nil, testMergeCfg, func(exporter string, err error) { + calls = append(calls, parseErrCall{exporter: exporter, err: err}) + }) var ops *dto.MetricFamily for _, f := range families { @@ -135,6 +179,52 @@ jumpstarter_operations_total{exporter="b",operation="off",result="success",drive if !exporters["exp-a"] || !exporters["exp-b"] { t.Errorf("exporters = %v, want exp-a and exp-b", exporters) } + if len(calls) != 1 { + t.Fatalf("parse-error callbacks = %d, want 1", len(calls)) + } + if calls[0].exporter != "bad" { + t.Errorf("parse-error exporter = %q, want bad", calls[0].exporter) + } + if calls[0].err == nil { + t.Fatal("parse-error callback err is nil") + } +} + +func TestMergeSnapshots_ReportsOpenMetricsExemplarParseError(t *testing.T) { + var gotExporter string + var gotErr error + families := mergeSnapshots([]exporterSnapshot{ + {name: "sidekick", text: []byte(pythonOpenMetricsWithExemplar)}, + }, nil, testMergeCfg, func(exporter string, err error) { + gotExporter = exporter + gotErr = err + }) + for _, f := range families { + if strings.HasPrefix(f.GetName(), "jumpstarter_operation_duration_seconds") { + t.Fatalf("unparseable snapshot must be omitted, got family %s", f.GetName()) + } + } + if gotExporter != "sidekick" { + t.Errorf("parse-error exporter = %q, want sidekick", gotExporter) + } + if gotErr == nil { + t.Fatal("parse-error callback err is nil") + } +} + +func TestRecordMetricsParseError_IncrementsLabeledCounter(t *testing.T) { + svc := &TelemetryService{} + svc.initScrapeTimeouts() + svc.recordMetricsParseError("sidekick", errors.New("parse failed")) + + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + value := labeledCounterValue(t, mfs, metricsParseErrorsMetric, labelExporter, "sidekick") + if value != 1 { + t.Fatalf("%s{exporter=sidekick} = %v, want 1", metricsParseErrorsMetric, value) + } } func TestEncodeMetricFamilies_WritesOpenMetricsEOF(t *testing.T) { diff --git a/controller/internal/service/metrics_stream_test.go b/controller/internal/service/metrics_stream_test.go index 6f4a50cd0..6aed15c36 100644 --- a/controller/internal/service/metrics_stream_test.go +++ b/controller/internal/service/metrics_stream_test.go @@ -400,3 +400,70 @@ jumpstarter_operations_total{exporter="spoofed",operation="on",result="success", default: } } + +func TestFanout_UnparseableSnapshotIncrementsParseErrorsOnSameResponse(t *testing.T) { + svc, client, addr := startTestHub(t, time.Second) + ctx := exporterStreamCtx(t, svc, "exporter:jumpstarter:sidekick:uid1") + stream, err := client.MetricsStream(ctx) + if err != nil { + t.Fatalf("MetricsStream: %v", err) + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_Register{ + Register: &pb.MetricsRegister{Identity: "sidekick"}, + }, + }); err != nil { + t.Fatalf("Send register: %v", err) + } + + errCh := make(chan error, 1) + go func() { + for { + msg, err := stream.Recv() + if err != nil { + errCh <- err + return + } + if msg.GetScrapeRequest() == nil { + continue + } + if err := stream.Send(&pb.MetricsStreamRequest{ + Msg: &pb.MetricsStreamRequest_ScrapeResponse{ + ScrapeResponse: &pb.MetricsScrapeResponse{MetricsText: []byte(pythonOpenMetricsWithExemplar)}, + }, + }); err != nil { + errCh <- err + return + } + } + }() + waitRegistered(t, svc, 1) + + code, body := httpGet(t, "http://"+addr+"/metrics") + if code != http.StatusOK { + t.Fatalf("GET /metrics status=%d body=%s", code, body) + } + if strings.Contains(body, "jumpstarter_operation_duration_seconds") { + t.Errorf("unparseable exporter snapshot must be omitted, body:\n%s", body) + } + if !strings.Contains(body, metricsParseErrorsMetric) || !strings.Contains(body, `exporter="sidekick"`) { + t.Errorf("parse-error counter missing from same /metrics response:\n%s", body) + } + + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + value := labeledCounterValue(t, mfs, metricsParseErrorsMetric, labelExporter, "sidekick") + if value < 1 { + t.Fatalf("%s{exporter=sidekick} = %v, want >= 1 body:\n%s", metricsParseErrorsMetric, value, body) + } + + select { + case err := <-errCh: + if err != nil && err != io.EOF && status.Code(err) != codes.Canceled && status.Code(err) != codes.Unavailable { + t.Fatalf("stream goroutine: %v", err) + } + default: + } +} diff --git a/controller/internal/service/telemetry_http.go b/controller/internal/service/telemetry_http.go index 6d12daff4..ecd403019 100644 --- a/controller/internal/service/telemetry_http.go +++ b/controller/internal/service/telemetry_http.go @@ -24,7 +24,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" ctrl "sigs.k8s.io/controller-runtime" ) @@ -40,7 +39,18 @@ func (s *TelemetryService) initScrapeTimeouts() { Name: scrapeTimeoutsMetric, Help: "Exporter MetricsStream scrapes that exceeded scrapeTimeout.", }) - s.metricsRegistry.MustRegister(s.scrapeTimeouts) + s.parseErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: metricsParseErrorsMetric, + Help: "Exporter MetricsStream snapshots omitted because OpenMetrics parse failed.", + }, []string{labelExporter}) + s.metricsRegistry.MustRegister(s.scrapeTimeouts, s.parseErrors) +} + +func (s *TelemetryService) recordMetricsParseError(exporter string, err error) { + if s.parseErrors != nil { + s.parseErrors.WithLabelValues(exporter).Inc() + } + ctrl.Log.WithName("telemetry").Error(err, "exporter metrics snapshot omitted", logFieldExporter, exporter) } func metricsHTTPEnabled(addr string) bool { @@ -110,19 +120,19 @@ func (s *TelemetryService) handleReadyz(w http.ResponseWriter, _ *http.Request) func (s *TelemetryService) handleMetrics(w http.ResponseWriter, r *http.Request) { s.initScrapeTimeouts() snaps := s.fanoutScrapes(r.Context()) + // Merge exporter snapshots first so parse failures increment parseErrors + // before hub metrics are gathered into the same /metrics response. + families := mergeSnapshots(snaps, nil, s.mergeConfigFor, s.recordMetricsParseError) - var extra []*dto.MetricFamily if s.metricsRegistry != nil { gathered, err := s.metricsRegistry.Gather() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - extra = gathered + families = mergeSnapshots(nil, append(gathered, families...), s.mergeConfigFor, nil) } - families := mergeSnapshots(snaps, extra, s.mergeConfigFor) - var buf bytes.Buffer if err := encodeMetricFamilies(&buf, families); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/controller/internal/service/telemetry_service.go b/controller/internal/service/telemetry_service.go index e3c60bb77..b2be84936 100644 --- a/controller/internal/service/telemetry_service.go +++ b/controller/internal/service/telemetry_service.go @@ -95,6 +95,7 @@ type TelemetryService struct { stateMu sync.Mutex conns map[string]*metricsConn scrapeTimeouts prometheus.Counter + parseErrors *prometheus.CounterVec metricsRegistry *prometheus.Registry metricsAddr string grpcReady atomic.Bool From 435eebee5a40c6a438a996bf99e22a661b753dc0 Mon Sep 17 00:00:00 2001 From: Roddie Kieley Date: Wed, 26 Aug 2026 20:44:31 -0230 Subject: [PATCH 3/5] feat: ship telemetry image, TLS, and metrics CR fields (JEP-0013 Phase 3) Build /telemetry into the controller image and have the operator mount cert-manager TLS, advertise the CA, and expose scrape flags plus the HTTP metrics port so reverse-scrape can run in-cluster. --- controller/Containerfile | 7 + controller/Makefile | 2 + .../api/v1alpha1/jumpstarter_types.go | 20 +++ .../api/v1alpha1/zz_generated.deepcopy.go | 31 ++++ ...operator.jumpstarter.dev_jumpstarters.yaml | 40 ++++- .../jumpstarter/jumpstarter_controller.go | 3 +- .../controller/jumpstarter/telemetry.go | 68 +++++++- .../jumpstarter/telemetry_phase3_test.go | 158 ++++++++++++++++++ .../controller/jumpstarter/telemetry_test.go | 89 ++++++++-- controller/image_layout_test.go | 62 +++++++ 10 files changed, 450 insertions(+), 30 deletions(-) create mode 100644 controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go create mode 100644 controller/image_layout_test.go diff --git a/controller/Containerfile b/controller/Containerfile index 846e4d340..d79593f21 100644 --- a/controller/Containerfile +++ b/controller/Containerfile @@ -48,11 +48,18 @@ RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1 go build -a \ -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ -o router ./cmd/router +RUN --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \ + --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a \ + -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \ + -o telemetry ./cmd/telemetry FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1786321990@sha256:7e7f79ab747bf2b452e3043dd89f388e92be4c7fdcc8b815b58adf6c99c39c95 WORKDIR / COPY --from=builder /build/manager . COPY --from=builder /build/router . +COPY --from=builder /build/telemetry . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/controller/Makefile b/controller/Makefile index 2d64ed9e0..efda65347 100644 --- a/controller/Makefile +++ b/controller/Makefile @@ -123,6 +123,7 @@ build-operator-ci: build: manifests generate fmt vet ## Build manager binary. go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go go build -ldflags "$(LDFLAGS)" -o bin/router ./cmd/router + go build -ldflags "$(LDFLAGS)" -o bin/telemetry ./cmd/telemetry go build -ldflags "$(LDFLAGS)" -o bin/exporter-set-controller cmd/exporter-set-controller/main.go .PHONY: run @@ -149,6 +150,7 @@ docker-build-ci: ## Build docker images from pre-compiled host binaries (fast CI rm -rf bin/ci-stage && mkdir -p bin/ci-stage/controller bin/ci-stage/esc CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/manager cmd/main.go CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/router ./cmd/router + CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/telemetry ./cmd/telemetry CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/esc/exporter-set-controller cmd/exporter-set-controller/main.go $(CONTAINER_TOOL) build --build-arg BIN=manager -t $(IMG) -f Containerfile.prebuilt bin/ci-stage/controller $(CONTAINER_TOOL) build --build-arg BIN=exporter-set-controller -t $(EXPORTER_SET_CONTROLLER_IMG) -f Containerfile.prebuilt bin/ci-stage/esc diff --git a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go index ae1271d1f..abc9cd95e 100644 --- a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -312,6 +312,10 @@ type TelemetryConfig struct { // gRPC configuration for the telemetry service. // Use this to configure TLS when not using cert-manager. GRPC TelemetryGRPCConfig `json:"grpc,omitempty"` + + // Metrics configures reverse-scrape fan-out and Prometheus exposition + // (JEP-0013). Loki and ServiceMonitor fields are later phases. + Metrics TelemetryMetricsConfig `json:"metrics,omitempty"` } // TelemetryGRPCConfig defines gRPC configuration for the telemetry service. @@ -325,6 +329,22 @@ type TelemetryGRPCConfig struct { TLS TLSConfig `json:"tls,omitempty"` } +// TelemetryMetricsConfig configures telemetry /metrics reverse-scrape behavior. +type TelemetryMetricsConfig struct { + // Allowlist of keys to include in Prometheus exemplars. Unlisted keys are omitted. + // +kubebuilder:default={"client","lease_id"} + ExemplarKeys []string `json:"exemplarKeys,omitempty"` + + // Allowed driver_type label values. Unlisted types are remapped to "other". + // +kubebuilder:default={"power","storage","network","serial","console","video","composite"} + DriverTypeEnum []string `json:"driverTypeEnum,omitempty"` + + // Max wait for parallel exporter MetricsStream responses during a /metrics fan-out. + // Should be lower than the Prometheus scrape_timeout. + // +kubebuilder:default="7s" + ScrapeTimeout *metav1.Duration `json:"scrapeTimeout,omitempty"` +} + // TelemetryLoggingConfig configures the log push path to the telemetry service. type TelemetryLoggingConfig struct { // Filter controls which log entries are forwarded to the telemetry service. diff --git a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 56f9f7d77..e7f233545 100644 --- a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -902,6 +902,7 @@ func (in *TelemetryConfig) DeepCopyInto(out *TelemetryConfig) { in.Resources.DeepCopyInto(&out.Resources) out.Logging = in.Logging out.GRPC = in.GRPC + in.Metrics.DeepCopyInto(&out.Metrics) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryConfig. @@ -960,3 +961,33 @@ func (in *TelemetryLoggingFilterConfig) DeepCopy() *TelemetryLoggingFilterConfig in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryMetricsConfig) DeepCopyInto(out *TelemetryMetricsConfig) { + *out = *in + if in.ExemplarKeys != nil { + in, out := &in.ExemplarKeys, &out.ExemplarKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DriverTypeEnum != nil { + in, out := &in.DriverTypeEnum, &out.DriverTypeEnum + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ScrapeTimeout != nil { + in, out := &in.ScrapeTimeout, &out.ScrapeTimeout + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryMetricsConfig. +func (in *TelemetryMetricsConfig) DeepCopy() *TelemetryMetricsConfig { + if in == nil { + return nil + } + out := new(TelemetryMetricsConfig) + in.DeepCopyInto(out) + return out +} diff --git a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index f5784b859..8ccdf97b5 100644 --- a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.18.0 + controller-gen.kubebuilder.io/version: v0.21.0 name: jumpstarters.operator.jumpstarter.dev spec: group: operator.jumpstarter.dev @@ -1087,7 +1087,8 @@ spec: Number of controller replicas to run. Currently only 1 replica is supported because the controller uses in-memory state for gRPC stream coordination (Dial/Listen). Values greater than 1 will - be clamped to 1 with a warning. See issue 1013 for HA controller support. + be clamped to 1 with a warning. See https://github.com/jumpstarter-dev/jumpstarter/issues/1013 + for the tracking issue on HA controller support. format: int32 minimum: 1 type: integer @@ -2163,6 +2164,41 @@ spec: type: string type: object type: object + metrics: + description: |- + Metrics configures reverse-scrape fan-out and Prometheus exposition + (JEP-0013). Loki and ServiceMonitor fields are later phases. + properties: + driverTypeEnum: + default: + - power + - storage + - network + - serial + - console + - video + - composite + description: Allowed driver_type label values. Unlisted types + are remapped to "other". + items: + type: string + type: array + exemplarKeys: + default: + - client + - lease_id + description: Allowlist of keys to include in Prometheus exemplars. + Unlisted keys are omitted. + items: + type: string + type: array + scrapeTimeout: + default: 7s + description: |- + Max wait for parallel exporter MetricsStream responses during a /metrics fan-out. + Should be lower than the Prometheus scrape_timeout. + type: string + type: object replicas: default: 1 description: |- diff --git a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 1a15ddf47..54899c9e8 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -1326,7 +1326,8 @@ func (r *JumpstarterReconciler) buildConfig(ctx context.Context, jumpstarter *op Keys: jumpstarter.Spec.DeprecatedLabels.Keys, } - // Telemetry configuration. + // Telemetry configuration. When cert-manager is enabled, inline the CA so + // GetServiceEndpoints.certificate lets exporters verify telemetry TLS. if jumpstarter.Spec.Telemetry != nil && jumpstarter.Spec.Telemetry.Enabled { t := jumpstarter.Spec.Telemetry telemetryCfg := &config.Telemetry{ diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go index c94945b17..3b0a92b36 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go @@ -19,6 +19,7 @@ package jumpstarter import ( "context" "fmt" + "strings" "time" certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -39,14 +40,23 @@ import ( const ( telemetryPort = 9093 + telemetryMetricsPort = 8080 telemetryCertSuffix = "-telemetry-tls" telemetryServiceName = "jumpstarter-telemetry" telemetryComponentApp = "jumpstarter-telemetry" telemetrySASuffix = "-telemetry" grpcPortName = "grpc" + metricsPortName = "metrics" telemetryCARequeueInterval = 30 * time.Second ) +// JEP-0013 default allowlists used when spec.telemetry.metrics omits a field. +var ( + defaultTelemetryDriverTypeEnum = []string{"power", "storage", "network", "serial", "console", "video", "composite"} + defaultTelemetryExemplarKeys = []string{"client", "lease_id"} + defaultTelemetryScrapeTimeout = 7 * time.Second +) + // reconcileTelemetryDeploymentStage reconciles only the telemetry Deployment (and cleanup). // It is called from the Deployment stage of the reconcile loop, before Services. // Certificate reconciliation is handled in certificates.go alongside other certs. @@ -251,6 +261,12 @@ func (r *JumpstarterReconciler) reconcileTelemetryService(ctx context.Context, j TargetPort: intstr.FromInt(telemetryPort), Protocol: corev1.ProtocolTCP, }, + { + Name: metricsPortName, + Port: int32(telemetryMetricsPort), + TargetPort: intstr.FromInt(telemetryMetricsPort), + Protocol: corev1.ProtocolTCP, + }, }, }, } @@ -293,6 +309,29 @@ func (r *JumpstarterReconciler) reconcileTelemetryService(ctx context.Context, j return nil } +// telemetryContainerArgs returns gRPC and HTTP flags for the telemetry binary. +func telemetryContainerArgs(t *operatorv1alpha1.TelemetryConfig) []string { + timeout := defaultTelemetryScrapeTimeout + if t.Metrics.ScrapeTimeout != nil && t.Metrics.ScrapeTimeout.Duration > 0 { + timeout = t.Metrics.ScrapeTimeout.Duration + } + enum := t.Metrics.DriverTypeEnum + if len(enum) == 0 { + enum = defaultTelemetryDriverTypeEnum + } + keys := t.Metrics.ExemplarKeys + if len(keys) == 0 { + keys = defaultTelemetryExemplarKeys + } + return []string{ + fmt.Sprintf("--grpc-bind=:%d", telemetryPort), + fmt.Sprintf("-metrics-bind-address=:%d", telemetryMetricsPort), + fmt.Sprintf("-scrape-timeout=%s", timeout), + fmt.Sprintf("-driver-type-enum=%s", strings.Join(enum, ",")), + fmt.Sprintf("-exemplar-keys=%s", strings.Join(keys, ",")), + } +} + // createTelemetryDeployment builds the desired Deployment for the telemetry service. // tlsSecretHash is included as a pod annotation to trigger rolling restarts on cert renewal. func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSecretHash string) *appsv1.Deployment { @@ -325,6 +364,10 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec }, }, }, + { + Name: "GRPC_TELEMETRY_ENDPOINT", + Value: telemetryEndpointFor(jumpstarter.Namespace), + }, } var volumeMounts []corev1.VolumeMount @@ -390,22 +433,27 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec Image: t.Image, ImagePullPolicy: t.ImagePullPolicy, Command: []string{"/telemetry"}, - Args: []string{ - fmt.Sprintf("--grpc-bind=:%d", telemetryPort), - }, - Env: envVars, - VolumeMounts: volumeMounts, + Args: telemetryContainerArgs(t), + Env: envVars, + VolumeMounts: volumeMounts, Ports: []corev1.ContainerPort{ { ContainerPort: int32(telemetryPort), Name: grpcPortName, Protocol: corev1.ProtocolTCP, }, + { + ContainerPort: int32(telemetryMetricsPort), + Name: metricsPortName, + Protocol: corev1.ProtocolTCP, + }, }, LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(telemetryPort), + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(telemetryMetricsPort), + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: 10, @@ -416,8 +464,10 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(telemetryPort), + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(telemetryMetricsPort), + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: 5, diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go new file mode 100644 index 000000000..76cbed5be --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_phase3_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/api/v1alpha1" +) + +// JEP-0013 operator defaults (Phase 3). Keep these literals in the tests so +// they encode the JEP rather than the implementation. +const ( + jepDefaultScrapeTimeoutFlag = "-scrape-timeout=7s" + jepDefaultDriverTypeEnumFlag = "-driver-type-enum=power,storage,network,serial,console,video,composite" + jepDefaultExemplarKeysFlag = "-exemplar-keys=client,lease_id" + jepMetricsBindFlag = "-metrics-bind-address=:8080" +) + +func phase3TelemetryJS(name, namespace string) *operatorv1alpha1.Jumpstarter { + return &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: false}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + } +} + +var _ = Describe("createTelemetryDeployment JEP-0013 Phase 3", func() { + It("passes default scrapeTimeout, driverTypeEnum, and exemplarKeys as flags", func() { + dep := createTelemetryDeployment(phase3TelemetryJS("js", "ns"), "") + c := dep.Spec.Template.Spec.Containers[0] + + Expect(c.Args).To(ContainElement(jepMetricsBindFlag)) + Expect(c.Args).To(ContainElement(jepDefaultScrapeTimeoutFlag)) + Expect(c.Args).To(ContainElement(jepDefaultDriverTypeEnumFlag)) + Expect(c.Args).To(ContainElement(jepDefaultExemplarKeysFlag)) + }) + + It("passes custom spec.telemetry.metrics fields as flags", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Metrics = operatorv1alpha1.TelemetryMetricsConfig{ + ExemplarKeys: []string{"client", "board-type"}, + DriverTypeEnum: []string{"power", "can"}, + ScrapeTimeout: &metav1.Duration{Duration: 3 * time.Second}, + } + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-scrape-timeout=3s")) + Expect(c.Args).To(ContainElement("-driver-type-enum=power,can")) + Expect(c.Args).To(ContainElement("-exemplar-keys=client,board-type")) + }) + + It("sets GRPC_TELEMETRY_ENDPOINT to the in-cluster telemetry Service", func() { + js := phase3TelemetryJS("js", "jumpstarter-lab") + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(envValue(c, "GRPC_TELEMETRY_ENDPOINT")).To(Equal(telemetryEndpointFor(js.Namespace))) + }) + + It("exposes container port metrics on 8080", func() { + c := createTelemetryDeployment(phase3TelemetryJS("js", "ns"), "").Spec.Template.Spec.Containers[0] + p := namedContainerPort(c.Ports, metricsPortName) + Expect(p).NotTo(BeNil(), "expected container port named metrics") + Expect(p.ContainerPort).To(Equal(int32(telemetryMetricsPort))) + Expect(p.Protocol).To(Equal(corev1.ProtocolTCP)) + }) +}) + +var _ = Describe("Telemetry ConfigMap certificate (JEP-0013 TLS)", func() { + const crName = "test-telemetry-ca" + + var crNamespace string + ctx := context.Background() + + BeforeEach(func() { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "telemetry-ca-"}} + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + crNamespace = ns.Name + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: crNamespace}, + }) + }) + + It("includes the CA PEM when cert-manager is enabled and the CA secret exists", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: true}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + }, + }, + } + Expect(k8sClient.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: crName + caCertificateSuffix, + Namespace: crNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte(testPEM)}, + })).To(Succeed()) + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + cfg, err := r.buildConfig(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Telemetry).NotTo(BeNil()) + Expect(cfg.Telemetry.Enabled).To(BeTrue()) + Expect(cfg.Telemetry.Endpoint).To(Equal(telemetryEndpointFor(crNamespace))) + Expect(cfg.Telemetry.Certificate).To(Equal(testPEM)) + }) + + It("omits the telemetry certificate when cert-manager is disabled", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: false}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + }, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + cfg, err := r.buildConfig(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Telemetry).NotTo(BeNil()) + Expect(cfg.Telemetry.Certificate).To(BeEmpty()) + }) +}) diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go index 77b2d2c25..978de4e4a 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go @@ -149,20 +149,27 @@ var _ = Describe("Telemetry Lifecycle", func() { Expect(deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot).NotTo(BeNil()) Expect(*deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot).To(BeTrue()) - By("verifying the Service exists") + By("verifying the Service exists with gRPC and metrics ports") svc := &corev1.Service{} Expect(k8sClient.Get(ctx, types.NamespacedName{ Name: telemetryServiceName, Namespace: crNamespace, }, svc)).To(Succeed()) Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) - Expect(svc.Spec.Ports).To(HaveLen(1)) - Expect(svc.Spec.Ports[0].Port).To(Equal(int32(telemetryPort))) - Expect(svc.Spec.Ports[0].Name).To(Equal("grpc")) Expect(svc.Spec.Selector).To(HaveKeyWithValue("app", "jumpstarter-telemetry")) + + grpcP := namedServicePort(svc.Spec.Ports, grpcPortName) + Expect(grpcP).NotTo(BeNil(), "expected Service port named grpc") + Expect(grpcP.Port).To(Equal(int32(telemetryPort))) + Expect(grpcP.TargetPort.IntValue()).To(Equal(telemetryPort)) + + metricsP := namedServicePort(svc.Spec.Ports, metricsPortName) + Expect(metricsP).NotTo(BeNil(), "expected Service port named metrics (JEP-0013 /metrics scrape)") + Expect(metricsP.Port).To(Equal(int32(telemetryMetricsPort))) + Expect(metricsP.TargetPort.IntValue()).To(Equal(telemetryMetricsPort)) }) - It("includes liveness and readiness probes on the telemetry container", func() { + It("uses HTTP /healthz and /readyz on the metrics port (JEP-0013 DD-7)", func() { By("creating a Jumpstarter CR with telemetry enabled") spec := makeJumpstarterSpec() spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ @@ -184,15 +191,17 @@ var _ = Describe("Telemetry Lifecycle", func() { container := deployment.Spec.Template.Spec.Containers[0] - By("verifying liveness probe uses TCP on the gRPC port") + By("verifying liveness probe is HTTP GET /healthz on the metrics port") Expect(container.LivenessProbe).NotTo(BeNil()) - Expect(container.LivenessProbe.TCPSocket).NotTo(BeNil()) - Expect(container.LivenessProbe.TCPSocket.Port.IntValue()).To(Equal(telemetryPort)) + Expect(container.LivenessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/healthz")) + Expect(container.LivenessProbe.HTTPGet.Port.IntValue()).To(Equal(telemetryMetricsPort)) - By("verifying readiness probe uses TCP on the gRPC port") + By("verifying readiness probe is HTTP GET /readyz on the metrics port") Expect(container.ReadinessProbe).NotTo(BeNil()) - Expect(container.ReadinessProbe.TCPSocket).NotTo(BeNil()) - Expect(container.ReadinessProbe.TCPSocket.Port.IntValue()).To(Equal(telemetryPort)) + Expect(container.ReadinessProbe.HTTPGet).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) + Expect(container.ReadinessProbe.HTTPGet.Port.IntValue()).To(Equal(telemetryMetricsPort)) }) It("respects the replicas field", func() { @@ -438,13 +447,14 @@ var _ = Describe("Telemetry Lifecycle", func() { container := dep.Spec.Template.Spec.Containers[0] - // Should have CONTROLLER_KEY + TLS env vars - Expect(container.Env).To(HaveLen(3)) + // CONTROLLER_KEY + GRPC_TELEMETRY_ENDPOINT + TLS env vars + Expect(container.Env).To(HaveLen(4)) envNames := make(map[string]string) for _, env := range container.Env { envNames[env.Name] = env.Value } Expect(envNames).To(HaveKey("CONTROLLER_KEY")) + Expect(envNames).To(HaveKey("GRPC_TELEMETRY_ENDPOINT")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_CERT_PEM", "/tls/tls.crt")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_KEY_PEM", "/tls/tls.key")) @@ -485,13 +495,14 @@ var _ = Describe("Telemetry Lifecycle", func() { container := dep.Spec.Template.Spec.Containers[0] - // Should have CONTROLLER_KEY + TLS env vars - Expect(container.Env).To(HaveLen(3)) + // CONTROLLER_KEY + GRPC_TELEMETRY_ENDPOINT + TLS env vars + Expect(container.Env).To(HaveLen(4)) envNames := make(map[string]string) for _, env := range container.Env { envNames[env.Name] = env.Value } Expect(envNames).To(HaveKey("CONTROLLER_KEY")) + Expect(envNames).To(HaveKey("GRPC_TELEMETRY_ENDPOINT")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_CERT_PEM", "/tls/tls.crt")) Expect(envNames).To(HaveKeyWithValue("EXTERNAL_KEY_PEM", "/tls/tls.key")) @@ -528,9 +539,15 @@ var _ = Describe("Telemetry Lifecycle", func() { container := dep.Spec.Template.Spec.Containers[0] - // Only CONTROLLER_KEY should be set (no TLS env vars) - Expect(container.Env).To(HaveLen(1)) - Expect(container.Env[0].Name).To(Equal("CONTROLLER_KEY")) + // CONTROLLER_KEY + GRPC_TELEMETRY_ENDPOINT (no TLS env vars) + Expect(container.Env).To(HaveLen(2)) + envNames := make(map[string]string) + for _, env := range container.Env { + envNames[env.Name] = env.Value + } + Expect(envNames).To(HaveKey("CONTROLLER_KEY")) + Expect(envNames).To(HaveKey("GRPC_TELEMETRY_ENDPOINT")) + Expect(envNames).NotTo(HaveKey("EXTERNAL_CERT_PEM")) // No volume mounts or volumes Expect(container.VolumeMounts).To(BeEmpty()) @@ -1066,3 +1083,39 @@ var _ = Describe("telemetryCANeedsRequeue", func() { Expect(r.telemetryCANeedsRequeue(ctx, js)).To(BeFalse()) }) }) + +func hasEnv(c corev1.Container, name string) bool { + for _, e := range c.Env { + if e.Name == name { + return true + } + } + return false +} + +func envValue(c corev1.Container, name string) string { + for _, e := range c.Env { + if e.Name == name { + return e.Value + } + } + return "" +} + +func namedContainerPort(ports []corev1.ContainerPort, name string) *corev1.ContainerPort { + for i := range ports { + if ports[i].Name == name { + return &ports[i] + } + } + return nil +} + +func namedServicePort(ports []corev1.ServicePort, name string) *corev1.ServicePort { + for i := range ports { + if ports[i].Name == name { + return &ports[i] + } + } + return nil +} diff --git a/controller/image_layout_test.go b/controller/image_layout_test.go new file mode 100644 index 000000000..0f6c541b2 --- /dev/null +++ b/controller/image_layout_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image_test + +import ( + "os" + "strings" + "testing" +) + +func mustRead(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func TestContainerfileBuildsTelemetryBinary(t *testing.T) { + content := mustRead(t, "Containerfile") + if !strings.Contains(content, "./cmd/telemetry") { + t.Error("Containerfile must compile ./cmd/telemetry") + } + if !strings.Contains(content, "-o telemetry") { + t.Error("Containerfile must emit a binary named telemetry") + } + if !strings.Contains(content, "COPY --from=builder /build/telemetry") { + t.Error("Containerfile must copy /build/telemetry into the runtime image as /telemetry") + } +} + +func TestMakefileBuildProducesTelemetry(t *testing.T) { + content := mustRead(t, "Makefile") + if !strings.Contains(content, "-o bin/telemetry") { + t.Error("make build must produce bin/telemetry") + } + if !strings.Contains(content, "cmd/telemetry") { + t.Error("make build must compile cmd/telemetry") + } +} + +func TestMakefileDockerBuildCIStagesTelemetry(t *testing.T) { + content := mustRead(t, "Makefile") + if !strings.Contains(content, "-o bin/ci-stage/controller/telemetry") { + t.Error("make docker-build-ci must stage telemetry next to manager and router so Containerfile.prebuilt COPY . . places /telemetry") + } +} From ac7e0835c79884df11bae0c0662ed03519c09874 Mon Sep 17 00:00:00 2001 From: Roddie Kieley Date: Wed, 26 Aug 2026 21:28:46 -0230 Subject: [PATCH 4/5] feat: Loki HTTP push with backpressure drop markers (JEP-0013 Phase 3) Forward PushLogs to Loki through a bounded ring buffer so a slow sink cannot stall exporters; overflow becomes one drop-marker LogEntry and jumpstarter_telemetry_dropped_total{destination="loki"}. --- controller/cmd/telemetry/main.go | 28 +- .../api/v1alpha1/jumpstarter_types.go | 42 +- .../api/v1alpha1/zz_generated.deepcopy.go | 48 ++ ...operator.jumpstarter.dev_jumpstarters.yaml | 43 +- .../controller/jumpstarter/telemetry.go | 75 +++- .../jumpstarter/telemetry_loki_test.go | 128 ++++++ controller/internal/service/loki_push.go | 425 ++++++++++++++++++ controller/internal/service/loki_push_test.go | 400 +++++++++++++++++ controller/internal/service/telemetry_http.go | 11 +- .../internal/service/telemetry_service.go | 117 +++-- 10 files changed, 1274 insertions(+), 43 deletions(-) create mode 100644 controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go create mode 100644 controller/internal/service/loki_push.go create mode 100644 controller/internal/service/loki_push_test.go diff --git a/controller/cmd/telemetry/main.go b/controller/cmd/telemetry/main.go index cf2266048..702a8744d 100644 --- a/controller/cmd/telemetry/main.go +++ b/controller/cmd/telemetry/main.go @@ -16,8 +16,8 @@ limitations under the License. // jumpstarter-telemetry reverse-scrapes exporter metrics via MetricsStream and // receives structured log entries via PushLogs. Logs are written to structured -// stdout for downstream log shippers (Promtail, Grafana Alloy, Vector). -// Loki push is a later Phase 3 PR. +// stdout for downstream log shippers (Promtail, Grafana Alloy, Vector) and +// optionally pushed to Loki's HTTP API when -loki-url is set. // // TLS: always enabled. Set EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM to file paths of // operator-mounted cert/key (e.g. from a cert-manager Secret); when absent a @@ -78,6 +78,10 @@ func main() { var scrapeTimeout time.Duration var driverTypeEnum string var exemplarKeys string + var lokiURL string + var lokiQueueDepth int + var lokiInsecure bool + var lokiCAFile string flag.StringVar(&bindAddr, "grpc-bind", ":9093", "TCP address to bind the gRPC server to") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "TCP address for HTTP GET /metrics, /healthz, and /readyz. Use 0 to disable.") @@ -87,6 +91,14 @@ func main() { "Comma-separated allowlist of driver_type values; others are remapped to other") flag.StringVar(&exemplarKeys, "exemplar-keys", strings.Join(service.DefaultExemplarKeys, ","), "Comma-separated allowlist of Prometheus exemplar keys") + flag.StringVar(&lokiURL, "loki-url", "", + "Loki HTTP push URL (optional; telemetry runs metrics-only when empty)") + flag.IntVar(&lokiQueueDepth, "loki-queue-depth", 10000, + "Ring buffer depth for Loki log push") + flag.BoolVar(&lokiInsecure, "loki-insecure-skip-verify", false, + "Disable TLS certificate verification for Loki (development/testing only)") + flag.StringVar(&lokiCAFile, "loki-ca-file", "", + "PEM CA bundle used to verify the Loki TLS endpoint") opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -124,6 +136,18 @@ func main() { DriverTypeEnum: splitCSV(driverTypeEnum), ExemplarKeys: splitCSV(exemplarKeys), Signer: signer, + LokiConfig: service.LokiConfig{ + URL: lokiURL, + Username: os.Getenv("LOKI_USERNAME"), + Password: os.Getenv("LOKI_PASSWORD"), + Token: os.Getenv("LOKI_TOKEN"), + CAFile: lokiCAFile, + InsecureSkipVerify: lokiInsecure, + QueueDepth: lokiQueueDepth, + }, + } + if lokiURL != "" { + logger.Info("Loki HTTP push configured", "url", lokiURL, "queueDepth", lokiQueueDepth) } // Register signal handler before starting the service so no signal diff --git a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go index abc9cd95e..c9198471f 100644 --- a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -314,8 +314,48 @@ type TelemetryConfig struct { GRPC TelemetryGRPCConfig `json:"grpc,omitempty"` // Metrics configures reverse-scrape fan-out and Prometheus exposition - // (JEP-0013). Loki and ServiceMonitor fields are later phases. + // (JEP-0013). ServiceMonitor fields are a later phase. Metrics TelemetryMetricsConfig `json:"metrics,omitempty"` + + // Loki configures optional HTTP push of ingested logs to a Loki-compatible + // endpoint. When url is empty, telemetry runs metrics-only. + Loki TelemetryLokiConfig `json:"loki,omitempty"` + + // Backpressure configures the Loki log push ring buffer. + Backpressure TelemetryBackpressureConfig `json:"backpressure,omitempty"` +} + +// TelemetryLokiConfig configures Loki HTTP push from the telemetry service. +type TelemetryLokiConfig struct { + // Loki push endpoint (http:// or https://). Optional — telemetry can run + // metrics-only without Loki. grpc:// is reserved but not implemented yet. + URL string `json:"url,omitempty"` + + // Secret with Loki credentials (username/password and/or token keys). + // See JEP-0013 DD-5: only the telemetry pod holds Loki credentials. + SecretRef string `json:"secretRef,omitempty"` + + // TLS settings for the Loki endpoint. + TLS TelemetryLokiTLSConfig `json:"tls,omitempty"` +} + +// TelemetryLokiTLSConfig configures TLS for the Loki push endpoint. +type TelemetryLokiTLSConfig struct { + // Secret containing a CA bundle (ca.crt key) to trust for the Loki endpoint. + CASecretRef string `json:"caSecretRef,omitempty"` + + // Disable TLS certificate verification (development/testing only). + // +kubebuilder:default=false + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` +} + +// TelemetryBackpressureConfig configures the Loki log push ring buffer. +type TelemetryBackpressureConfig struct { + // Ring buffer depth for Loki log push. On overflow, dropped entries are + // replaced by a single drop-marker LogEntry. + // +kubebuilder:default=10000 + // +kubebuilder:validation:Minimum=1 + QueueDepth int32 `json:"queueDepth,omitempty"` } // TelemetryGRPCConfig defines gRPC configuration for the telemetry service. diff --git a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index e7f233545..d28d4d6ea 100644 --- a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -891,6 +891,21 @@ func (in *TLSConfig) DeepCopy() *TLSConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryBackpressureConfig) DeepCopyInto(out *TelemetryBackpressureConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryBackpressureConfig. +func (in *TelemetryBackpressureConfig) DeepCopy() *TelemetryBackpressureConfig { + if in == nil { + return nil + } + out := new(TelemetryBackpressureConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TelemetryConfig) DeepCopyInto(out *TelemetryConfig) { *out = *in @@ -903,6 +918,8 @@ func (in *TelemetryConfig) DeepCopyInto(out *TelemetryConfig) { out.Logging = in.Logging out.GRPC = in.GRPC in.Metrics.DeepCopyInto(&out.Metrics) + out.Loki = in.Loki + out.Backpressure = in.Backpressure } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryConfig. @@ -962,6 +979,37 @@ func (in *TelemetryLoggingFilterConfig) DeepCopy() *TelemetryLoggingFilterConfig return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryLokiConfig) DeepCopyInto(out *TelemetryLokiConfig) { + *out = *in + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryLokiConfig. +func (in *TelemetryLokiConfig) DeepCopy() *TelemetryLokiConfig { + if in == nil { + return nil + } + out := new(TelemetryLokiConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryLokiTLSConfig) DeepCopyInto(out *TelemetryLokiTLSConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryLokiTLSConfig. +func (in *TelemetryLokiTLSConfig) DeepCopy() *TelemetryLokiTLSConfig { + if in == nil { + return nil + } + out := new(TelemetryLokiTLSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TelemetryMetricsConfig) DeepCopyInto(out *TelemetryMetricsConfig) { *out = *in diff --git a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 8ccdf97b5..1b4b7ccc2 100644 --- a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -2100,6 +2100,18 @@ spec: structured log entries from exporters via gRPC. The controller advertises the telemetry endpoint to exporters so they can push logs without cluster credentials. properties: + backpressure: + description: Backpressure configures the Loki log push ring buffer. + properties: + queueDepth: + default: 10000 + description: |- + Ring buffer depth for Loki log push. On overflow, dropped entries are + replaced by a single drop-marker LogEntry. + format: int32 + minimum: 1 + type: integer + type: object enabled: default: false description: |- @@ -2164,10 +2176,39 @@ spec: type: string type: object type: object + loki: + description: |- + Loki configures optional HTTP push of ingested logs to a Loki-compatible + endpoint. When url is empty, telemetry runs metrics-only. + properties: + secretRef: + description: |- + Secret with Loki credentials (username/password and/or token keys). + See JEP-0013 DD-5: only the telemetry pod holds Loki credentials. + type: string + tls: + description: TLS settings for the Loki endpoint. + properties: + caSecretRef: + description: Secret containing a CA bundle (ca.crt key) + to trust for the Loki endpoint. + type: string + insecureSkipVerify: + default: false + description: Disable TLS certificate verification (development/testing + only). + type: boolean + type: object + url: + description: |- + Loki push endpoint (http:// or https://). Optional — telemetry can run + metrics-only without Loki. grpc:// is reserved but not implemented yet. + type: string + type: object metrics: description: |- Metrics configures reverse-scrape fan-out and Prometheus exposition - (JEP-0013). Loki and ServiceMonitor fields are later phases. + (JEP-0013). ServiceMonitor fields are a later phase. properties: driverTypeEnum: default: diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go index 3b0a92b36..e06d3b21d 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go @@ -55,6 +55,10 @@ var ( defaultTelemetryDriverTypeEnum = []string{"power", "storage", "network", "serial", "console", "video", "composite"} defaultTelemetryExemplarKeys = []string{"client", "lease_id"} defaultTelemetryScrapeTimeout = 7 * time.Second + defaultTelemetryLokiQueueDepth = int32(10000) + lokiCAMountPath = "/loki-ca" + lokiCAFilePath = "/loki-ca/ca.crt" + lokiCAVolumeName = "loki-ca" ) // reconcileTelemetryDeploymentStage reconciles only the telemetry Deployment (and cleanup). @@ -323,13 +327,28 @@ func telemetryContainerArgs(t *operatorv1alpha1.TelemetryConfig) []string { if len(keys) == 0 { keys = defaultTelemetryExemplarKeys } - return []string{ + args := []string{ fmt.Sprintf("--grpc-bind=:%d", telemetryPort), fmt.Sprintf("-metrics-bind-address=:%d", telemetryMetricsPort), fmt.Sprintf("-scrape-timeout=%s", timeout), fmt.Sprintf("-driver-type-enum=%s", strings.Join(enum, ",")), fmt.Sprintf("-exemplar-keys=%s", strings.Join(keys, ",")), } + if t.Loki.URL != "" { + args = append(args, fmt.Sprintf("-loki-url=%s", t.Loki.URL)) + depth := t.Backpressure.QueueDepth + if depth <= 0 { + depth = defaultTelemetryLokiQueueDepth + } + args = append(args, fmt.Sprintf("-loki-queue-depth=%d", depth)) + if t.Loki.TLS.InsecureSkipVerify { + args = append(args, "-loki-insecure-skip-verify=true") + } + if t.Loki.TLS.CASecretRef != "" { + args = append(args, fmt.Sprintf("-loki-ca-file=%s", lokiCAFilePath)) + } + } + return args } // createTelemetryDeployment builds the desired Deployment for the telemetry service. @@ -398,6 +417,60 @@ func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter, tlsSec }) } + if t.Loki.URL != "" && t.Loki.SecretRef != "" { + optional := ptr.To(true) + envVars = append(envVars, + corev1.EnvVar{ + Name: "LOKI_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: t.Loki.SecretRef}, + Key: "username", + Optional: optional, + }, + }, + }, + corev1.EnvVar{ + Name: "LOKI_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: t.Loki.SecretRef}, + Key: "password", + Optional: optional, + }, + }, + }, + corev1.EnvVar{ + Name: "LOKI_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: t.Loki.SecretRef}, + Key: "token", + Optional: optional, + }, + }, + }, + ) + } + + if t.Loki.URL != "" && t.Loki.TLS.CASecretRef != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: lokiCAVolumeName, + MountPath: lokiCAMountPath, + ReadOnly: true, + }) + defaultMode := int32(420) + volumes = append(volumes, corev1.Volume{ + Name: lokiCAVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: t.Loki.TLS.CASecretRef, + DefaultMode: &defaultMode, + }, + }, + }) + } + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-telemetry", jumpstarter.Name), diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go new file mode 100644 index 000000000..66cdc5018 --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + 10|Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +const ( + jepDefaultLokiQueueDepthFlag = "-loki-queue-depth=10000" + jepLokiCAFileFlag = "-loki-ca-file=/loki-ca/ca.crt" +) + +var _ = Describe("createTelemetryDeployment JEP-0013 Loki push", func() { + It("omits Loki flags when spec.telemetry.loki.url is unset (metrics-only)", func() { + c := createTelemetryDeployment(phase3TelemetryJS("js", "ns"), "").Spec.Template.Spec.Containers[0] + for _, arg := range c.Args { + Expect(arg).NotTo(HavePrefix("-loki-url=")) + Expect(arg).NotTo(HavePrefix("-loki-queue-depth=")) + Expect(arg).NotTo(HavePrefix("-loki-ca-file=")) + Expect(arg).NotTo(HavePrefix("-loki-insecure-skip-verify")) + } + Expect(hasEnv(c, "LOKI_USERNAME")).To(BeFalse()) + Expect(hasEnv(c, "LOKI_PASSWORD")).To(BeFalse()) + Expect(hasEnv(c, "LOKI_TOKEN")).To(BeFalse()) + }) + + It("passes loki.url and default queueDepth when Loki is configured", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki-gateway.monitoring.svc:3100/loki/api/v1/push" + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-loki-url=https://loki-gateway.monitoring.svc:3100/loki/api/v1/push")) + Expect(c.Args).To(ContainElement(jepDefaultLokiQueueDepthFlag)) + }) + + It("passes custom spec.telemetry.backpressure.queueDepth", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "http://loki:3100" + js.Spec.Telemetry.Backpressure.QueueDepth = 20000 + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-loki-queue-depth=20000")) + }) + + It("wires Loki credentials from spec.telemetry.loki.secretRef", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki:3100/loki/api/v1/push" + js.Spec.Telemetry.Loki.SecretRef = "loki-credentials" + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(envSecretRef(c, "LOKI_USERNAME")).To(Equal(secretKeyRef{"loki-credentials", "username"})) + Expect(envSecretRef(c, "LOKI_PASSWORD")).To(Equal(secretKeyRef{"loki-credentials", "password"})) + Expect(envSecretRef(c, "LOKI_TOKEN")).To(Equal(secretKeyRef{"loki-credentials", "token"})) + }) + + It("mounts loki.tls.caSecretRef and passes -loki-ca-file", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki:3100/loki/api/v1/push" + js.Spec.Telemetry.Loki.TLS.CASecretRef = "loki-ca-bundle" + + dep := createTelemetryDeployment(js, "") + c := dep.Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement(jepLokiCAFileFlag)) + + var mount *corev1.VolumeMount + for i := range c.VolumeMounts { + if c.VolumeMounts[i].Name == "loki-ca" { + mount = &c.VolumeMounts[i] + } + } + Expect(mount).NotTo(BeNil(), "expected loki-ca volume mount") + Expect(mount.MountPath).To(Equal("/loki-ca")) + Expect(mount.ReadOnly).To(BeTrue()) + + var vol *corev1.Volume + for i := range dep.Spec.Template.Spec.Volumes { + if dep.Spec.Template.Spec.Volumes[i].Name == "loki-ca" { + vol = &dep.Spec.Template.Spec.Volumes[i] + } + } + Expect(vol).NotTo(BeNil()) + Expect(vol.Secret).NotTo(BeNil()) + Expect(vol.Secret.SecretName).To(Equal("loki-ca-bundle")) + }) + + It("passes -loki-insecure-skip-verify when tls.insecureSkipVerify is set", func() { + js := phase3TelemetryJS("js", "ns") + js.Spec.Telemetry.Loki.URL = "https://loki:3100/loki/api/v1/push" + js.Spec.Telemetry.Loki.TLS.InsecureSkipVerify = true + + c := createTelemetryDeployment(js, "").Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-loki-insecure-skip-verify=true")) + }) +}) + +type secretKeyRef struct { + name string + key string +} + +func envSecretRef(c corev1.Container, name string) secretKeyRef { + for _, e := range c.Env { + if e.Name == name && e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil { + return secretKeyRef{ + name: e.ValueFrom.SecretKeyRef.Name, + key: e.ValueFrom.SecretKeyRef.Key, + } + } + } + return secretKeyRef{} +} diff --git a/controller/internal/service/loki_push.go b/controller/internal/service/loki_push.go new file mode 100644 index 000000000..e1c929725 --- /dev/null +++ b/controller/internal/service/loki_push.go @@ -0,0 +1,425 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultLokiQueueDepth = 10000 + droppedTotalMetric = "jumpstarter_telemetry_dropped_total" + droppedDestination = "loki" + lokiPushPath = "/loki/api/v1/push" + lokiFlushInterval = 500 * time.Millisecond + lokiHTTPTimeout = 10 * time.Second +) + +// LokiConfig is the telemetry-side Loki HTTP push configuration. +type LokiConfig struct { + URL string + Username string + Password string + Token string + CAFile string + InsecureSkipVerify bool + QueueDepth int +} + +type lokiPushPayload struct { + Streams []lokiStream `json:"streams"` +} + +type lokiStream struct { + Stream map[string]string `json:"stream"` + Values [][]string `json:"values"` +} + +// LokiPusher is a bounded ring buffer that POSTs LogEntry batches to Loki. +type LokiPusher struct { + url string + username string + password string + token string + client *http.Client + queueDepth int + dropped prometheus.Counter + now func() time.Time + + mu sync.Mutex + entries []*pb.LogEntry + dropCount int + dropFirst time.Time +} + +func normalizeQueueDepth(d int) int { + if d <= 0 { + return defaultLokiQueueDepth + } + if d < 2 { + return 2 + } + return d +} + +func normalizeLokiPushURL(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("loki url: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("loki url: unsupported scheme %q (use http or https)", u.Scheme) + } + if u.Path == "" || u.Path == "/" { + u.Path = lokiPushPath + } + return u.String(), nil +} + +func lokiHTTPClient(cfg LokiConfig) (*http.Client, error) { + transport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return &http.Client{Timeout: lokiHTTPTimeout}, nil + } + transport = transport.Clone() + tlsCfg := &tls.Config{MinVersion: tls.VersionTLS12} + customTLS := false + if cfg.InsecureSkipVerify { + tlsCfg.InsecureSkipVerify = true + customTLS = true + } + if cfg.CAFile != "" { + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return nil, fmt.Errorf("loki ca file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("loki ca file: no certificates found in %s", cfg.CAFile) + } + tlsCfg.RootCAs = pool + customTLS = true + } + if customTLS { + transport.TLSClientConfig = tlsCfg + } + return &http.Client{Timeout: lokiHTTPTimeout, Transport: transport}, nil +} + +// NewLokiPusher returns nil, nil when URL is empty (metrics-only). grpc:// is rejected. +func NewLokiPusher(cfg LokiConfig, dropped *prometheus.CounterVec) (*LokiPusher, error) { + raw := strings.TrimSpace(cfg.URL) + if raw == "" { + return nil, nil + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("loki url: %w", err) + } + if u.Scheme == "grpc" || u.Scheme == "grpcs" { + return nil, fmt.Errorf("loki gRPC scheme %q is not implemented; use http:// or https://", u.Scheme) + } + pushURL, err := normalizeLokiPushURL(raw) + if err != nil { + return nil, err + } + client, err := lokiHTTPClient(cfg) + if err != nil { + return nil, err + } + var counter prometheus.Counter + if dropped != nil { + counter = dropped.WithLabelValues(droppedDestination) + } + return &LokiPusher{ + url: pushURL, + username: cfg.Username, + password: cfg.Password, + token: cfg.Token, + client: client, + queueDepth: normalizeQueueDepth(cfg.QueueDepth), + dropped: counter, + now: time.Now, + }, nil +} + +func (p *LokiPusher) maxReal() int { + n := p.queueDepth - 1 + if n < 1 { + return 1 + } + return n +} + +// Enqueue is non-blocking. On overflow, entries are replaced by a single drop marker. +func (p *LokiPusher) Enqueue(entry *pb.LogEntry) { + if p == nil || entry == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if len(p.entries) < p.maxReal() { + p.entries = append(p.entries, entry) + return + } + if p.dropCount == 0 { + p.dropFirst = p.now() + } + p.dropCount++ + if p.dropped != nil { + p.dropped.Inc() + } +} + +func makeDropMarker(count int, first, now time.Time) *pb.LogEntry { + window := 0 + if !first.IsZero() { + window = int(now.Sub(first).Seconds()) + } + return &pb.LogEntry{ + Timestamp: timestamppb.New(now), + Severity: "warning", + Message: "log entries dropped due to Loki backpressure", + Component: "telemetry", + Operation: "backpressure", + ExtraFields: map[string]string{ + "count": strconv.Itoa(count), + "window_seconds": strconv.Itoa(window), + }, + } +} + +func (p *LokiPusher) dropMarkerLocked() *pb.LogEntry { + return makeDropMarker(p.dropCount, p.dropFirst, p.now()) +} + +func (p *LokiPusher) queued() []*pb.LogEntry { + p.mu.Lock() + defer p.mu.Unlock() + out := append([]*pb.LogEntry(nil), p.entries...) + if p.dropCount > 0 { + out = append(out, p.dropMarkerLocked()) + } + return out +} + +func (p *LokiPusher) takeBatch() (entries []*pb.LogEntry, dropCount int, dropFirst time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + entries = p.entries + p.entries = nil + dropCount = p.dropCount + dropFirst = p.dropFirst + p.dropCount = 0 + p.dropFirst = time.Time{} + return entries, dropCount, dropFirst +} + +func (p *LokiPusher) restoreBatch(entries []*pb.LogEntry, dropCount int, dropFirst time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + p.entries = append(entries, p.entries...) + if dropCount > 0 { + if p.dropCount == 0 || (!dropFirst.IsZero() && (p.dropFirst.IsZero() || dropFirst.Before(p.dropFirst))) { + p.dropFirst = dropFirst + } + p.dropCount += dropCount + } +} + +// Flush POSTs queued entries to Loki. On HTTP failure the buffer is restored. +func (p *LokiPusher) Flush(ctx context.Context) error { + if p == nil { + return nil + } + entries, dropCount, dropFirst := p.takeBatch() + batch := append([]*pb.LogEntry(nil), entries...) + if dropCount > 0 { + batch = append(batch, makeDropMarker(dropCount, dropFirst, p.now())) + } + if len(batch) == 0 { + return nil + } + if err := p.push(ctx, batch); err != nil { + p.restoreBatch(entries, dropCount, dropFirst) + return err + } + return nil +} + +func (p *LokiPusher) push(ctx context.Context, entries []*pb.LogEntry) error { + body, err := json.Marshal(buildLokiPayload(entries, p.now())) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if p.token != "" { + req.Header.Set("Authorization", "Bearer "+p.token) + } else if p.username != "" || p.password != "" { + req.SetBasicAuth(p.username, p.password) + } + resp, err := p.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + slurp, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return fmt.Errorf("loki push %s: %s", resp.Status, bytes.TrimSpace(slurp)) + } + return nil +} + +// Run flushes the buffer periodically until ctx is cancelled, then flushes once more. +func (p *LokiPusher) Run(ctx context.Context) { + if p == nil { + return + } + ticker := time.NewTicker(lokiFlushInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + flushCtx, cancel := context.WithTimeout(context.Background(), lokiHTTPTimeout) + _ = p.Flush(flushCtx) + cancel() + return + case <-ticker.C: + _ = p.Flush(ctx) + } + } +} + +func buildLokiPayload(entries []*pb.LogEntry, now time.Time) lokiPushPayload { + type acc struct { + labels map[string]string + values [][]string + } + order := make([]string, 0) + grouped := make(map[string]*acc) + for _, e := range entries { + labels := lokiStreamLabels(e) + key := labels["component"] + "\x00" + labels["exporter"] + "\x00" + labels["namespace"] + g, ok := grouped[key] + if !ok { + g = &acc{labels: labels} + grouped[key] = g + order = append(order, key) + } + g.values = append(g.values, []string{unixNanoString(e, now), logLineJSON(e)}) + } + streams := make([]lokiStream, 0, len(order)) + for _, key := range order { + g := grouped[key] + streams = append(streams, lokiStream{Stream: g.labels, Values: g.values}) + } + return lokiPushPayload{Streams: streams} +} + +func lokiStreamLabels(e *pb.LogEntry) map[string]string { + m := make(map[string]string, 3) + if e.Component != "" { + m["component"] = e.Component + } + if e.Exporter != "" { + m["exporter"] = e.Exporter + } + if e.Namespace != "" { + m["namespace"] = e.Namespace + } + return m +} + +func unixNanoString(e *pb.LogEntry, now time.Time) string { + ts := now + if e.Timestamp != nil && e.Timestamp.IsValid() { + ts = e.Timestamp.AsTime() + } + return strconv.FormatInt(ts.UnixNano(), 10) +} + +func logLineJSON(e *pb.LogEntry) string { + body := map[string]string{} + if e.Message != "" { + body["msg"] = e.Message + } + if e.Severity != "" { + body["severity"] = e.Severity + } + if e.Component != "" { + body["component"] = e.Component + } + if e.Exporter != "" { + body["exporter"] = e.Exporter + } + if e.Namespace != "" { + body["namespace"] = e.Namespace + } + if e.Lease != "" { + body["lease"] = e.Lease + } + if e.Client != "" { + body["client"] = e.Client + } + if e.Operation != "" { + body["operation"] = e.Operation + } + if e.Result != "" { + body["result"] = e.Result + } + if e.DriverType != "" { + body["driver_type"] = e.DriverType + } + if e.Timestamp != nil && e.Timestamp.IsValid() { + body["ts"] = e.Timestamp.AsTime().Format(time.RFC3339Nano) + } + for k, v := range e.ExtraFields { + if _, reserved := reservedExtraFieldKeys[k]; reserved { + continue + } + if k == "" { + continue + } + body[k] = v + } + b, err := json.Marshal(body) + if err != nil { + return `{"msg":"marshal error"}` + } + return string(b) +} diff --git a/controller/internal/service/loki_push_test.go b/controller/internal/service/loki_push_test.go new file mode 100644 index 000000000..0122aa9ee --- /dev/null +++ b/controller/internal/service/loki_push_test.go @@ -0,0 +1,400 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + 10|Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func testDroppedCounter() *prometheus.CounterVec { + return prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: droppedTotalMetric, + Help: "Log entries dropped due to Loki backpressure.", + }, []string{"destination"}) +} + +func logEntry(msg string) *pb.LogEntry { + return &pb.LogEntry{ + Timestamp: timestamppb.New(time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC)), + Severity: "info", + Message: msg, + Component: "exporter", + Exporter: "lab-1", + Namespace: "jumpstarter", + Lease: "lease-1", + Client: "ci", + Operation: "flash", + Result: "success", + } +} + +func TestNewLokiPusher_EmptyURLIsNil(t *testing.T) { + p, err := NewLokiPusher(LokiConfig{}, testDroppedCounter()) + if err != nil { + t.Fatalf("empty URL: %v", err) + } + if p != nil { + t.Fatal("metrics-only deployments must not construct a Loki pusher") + } +} + +func TestNewLokiPusher_GRPCSchemeNotImplemented(t *testing.T) { + p, err := NewLokiPusher(LokiConfig{URL: "grpc://loki.monitoring.svc:9095"}, testDroppedCounter()) + if p != nil { + t.Fatal("gRPC Loki must not start an HTTP pusher") + } + if err == nil { + t.Fatal("expected error for unimplemented grpc:// Loki URL") + } +} + +func TestNormalizeLokiPushURL_AppendsPath(t *testing.T) { + got, err := normalizeLokiPushURL("http://loki:3100") + if err != nil { + t.Fatal(err) + } + if got != "http://loki:3100/loki/api/v1/push" { + t.Errorf("got %q", got) + } +} + +func TestNormalizeLokiPushURL_KeepsExistingPath(t *testing.T) { + raw := "https://loki-gateway.monitoring.svc:3100/loki/api/v1/push" + got, err := normalizeLokiPushURL(raw) + if err != nil { + t.Fatal(err) + } + if got != raw { + t.Errorf("got %q", got) + } +} + +func TestDefaultQueueDepthIs10000(t *testing.T) { + if d := normalizeQueueDepth(0); d != defaultLokiQueueDepth { + t.Errorf("normalizeQueueDepth(0) = %d, want %d", d, defaultLokiQueueDepth) + } + if defaultLokiQueueDepth != 10000 { + t.Errorf("default Loki queue depth = %d, want 10000 (JEP-0013)", defaultLokiQueueDepth) + } +} + +func TestLokiBuffer_OverflowEmitsDropMarker(t *testing.T) { + dropped := testDroppedCounter() + p, err := NewLokiPusher(LokiConfig{URL: "http://127.0.0.1:1", QueueDepth: 3}, dropped) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 26, 12, 0, 0, 0, time.UTC) + p.now = func() time.Time { return now } + + p.Enqueue(logEntry("a")) + p.Enqueue(logEntry("b")) + p.Enqueue(logEntry("c")) // overflow: reserved marker slot + now = now.Add(12 * time.Second) + p.Enqueue(logEntry("d")) // accumulate into the same marker + + queued := p.queued() + if len(queued) != 3 { + t.Fatalf("queued = %d, want 2 real entries + 1 drop marker", len(queued)) + } + if queued[0].Message != "a" || queued[1].Message != "b" { + t.Fatalf("real entries = %q, %q; want a, b", queued[0].Message, queued[1].Message) + } + marker := queued[2] + if marker.Severity != "warning" { + t.Errorf("marker severity = %q, want warning", marker.Severity) + } + if marker.Component != "telemetry" { + t.Errorf("marker component = %q, want telemetry", marker.Component) + } + if marker.Operation != "backpressure" { + t.Errorf("marker operation = %q, want backpressure", marker.Operation) + } + if marker.ExtraFields["count"] != "2" { + t.Errorf("marker count = %q, want 2", marker.ExtraFields["count"]) + } + if marker.ExtraFields["window_seconds"] != "12" { + t.Errorf("marker window_seconds = %q, want 12", marker.ExtraFields["window_seconds"]) + } + + got := counterValue(t, dropped, "loki") + if got != 2 { + t.Errorf("dropped_total{destination=loki} = %v, want 2", got) + } +} + +func TestLokiPush_HTTPPostsOpenMetricsCompatibleJSON(t *testing.T) { + var ( + mu sync.Mutex + gotPath string + gotAuth string + gotCT string + gotBody []byte + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + gotCT = r.Header.Get("Content-Type") + gotBody = body + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + dropped := testDroppedCounter() + p, err := NewLokiPusher(LokiConfig{ + URL: srv.URL, + Username: "loki", + Password: "secret", + QueueDepth: 10, + }, dropped) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("hello loki")) + if err := p.Flush(context.Background()); err != nil { + t.Fatalf("Flush: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if gotPath != "/loki/api/v1/push" { + t.Errorf("path = %q, want /loki/api/v1/push", gotPath) + } + if !strings.HasPrefix(gotCT, "application/json") { + t.Errorf("Content-Type = %q, want application/json", gotCT) + } + if !strings.HasPrefix(gotAuth, "Basic ") { + t.Errorf("Authorization = %q, want Basic", gotAuth) + } + + var payload lokiPushPayload + if err := json.Unmarshal(gotBody, &payload); err != nil { + t.Fatalf("unmarshal %s: %v", gotBody, err) + } + if len(payload.Streams) != 1 { + t.Fatalf("streams = %d, want 1", len(payload.Streams)) + } + stream := payload.Streams[0].Stream + if stream["component"] != "exporter" || stream["exporter"] != "lab-1" || stream["namespace"] != "jumpstarter" { + t.Errorf("stream labels = %v, want component/exporter/namespace", stream) + } + for _, highCard := range []string{"client", "lease", "lease_id", "operation"} { + if _, ok := stream[highCard]; ok { + t.Errorf("high-cardinality field %q must not be a Loki stream label", highCard) + } + } + if len(payload.Streams[0].Values) != 1 { + t.Fatalf("values = %d, want 1", len(payload.Streams[0].Values)) + } + ts, line := payload.Streams[0].Values[0][0], payload.Streams[0].Values[0][1] + if _, err := strconv.ParseInt(ts, 10, 64); err != nil { + t.Errorf("timestamp %q is not unix nanoseconds", ts) + } + var body map[string]string + if err := json.Unmarshal([]byte(line), &body); err != nil { + t.Fatalf("log line JSON: %v", err) + } + if body["msg"] != "hello loki" { + t.Errorf("msg = %q", body["msg"]) + } + if body["severity"] != "info" { + t.Errorf("severity = %q", body["severity"]) + } + if body["client"] != "ci" || body["lease"] != "lease-1" { + t.Errorf("high-cardinality fields missing from JSON body: %v", body) + } +} + +func TestLokiPush_FlushFailureKeepsBuffer(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + + p, err := NewLokiPusher(LokiConfig{URL: srv.URL, QueueDepth: 10}, testDroppedCounter()) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("keep me")) + if err := p.Flush(context.Background()); err == nil { + t.Fatal("expected flush error") + } + if len(p.queued()) != 1 { + t.Fatalf("failed flush must keep the buffer, queued = %d", len(p.queued())) + } +} + +func TestLokiPush_BearerToken(t *testing.T) { + var gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + p, err := NewLokiPusher(LokiConfig{URL: srv.URL, Token: "s3cret", QueueDepth: 8}, testDroppedCounter()) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("tok")) + if err := p.Flush(context.Background()); err != nil { + t.Fatal(err) + } + if gotAuth != "Bearer s3cret" { + t.Errorf("Authorization = %q, want Bearer s3cret", gotAuth) + } +} + +func TestDroppedTotalRegisteredOnMetrics(t *testing.T) { + svc := &TelemetryService{} + svc.initScrapeTimeouts() + mfs, err := svc.metricsRegistry.Gather() + if err != nil { + t.Fatal(err) + } + var found bool + var dest string + for _, mf := range mfs { + if mf.GetName() != droppedTotalMetric { + continue + } + found = true + if len(mf.Metric) > 0 { + for _, lp := range mf.Metric[0].GetLabel() { + if lp.GetName() == "destination" { + dest = lp.GetValue() + } + } + } + } + if !found { + t.Fatalf("%s missing from /metrics registry", droppedTotalMetric) + } + if dest != "loki" { + t.Errorf("destination label = %q, want loki", dest) + } +} + +func TestPushLogs_ForwardsAcceptedEntriesToLoki(t *testing.T) { + var gotBody []byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + signer := testSigner(t) + dropped := testDroppedCounter() + pusher, err := NewLokiPusher(LokiConfig{URL: srv.URL, QueueDepth: 10}, dropped) + if err != nil { + t.Fatal(err) + } + svc := &TelemetryService{BindAddr: ":0", Signer: signer, Loki: pusher} + ctx := authedCtx(t, signer, "exporter:jumpstarter:real-exporter:uid1") + + resp, err := svc.PushLogs(ctx, &pb.PushLogsRequest{Entries: []*pb.LogEntry{{ + Severity: "info", + Message: "from exporter", + Component: "exporter", + Exporter: "spoofed", + Namespace: "jumpstarter", + }}}) + if err != nil { + t.Fatal(err) + } + if resp.Dropped != 1 { + t.Fatalf("Dropped = %d, want 1 (identity mismatch is not Loki backpressure)", resp.Dropped) + } + if n := len(pusher.queued()); n != 0 { + t.Fatalf("identity-mismatch entries must not be queued for Loki, queued=%d", n) + } + + resp, err = svc.PushLogs(ctx, &pb.PushLogsRequest{Entries: []*pb.LogEntry{{ + Severity: "info", + Message: "ok", + Component: "exporter", + Exporter: "real-exporter", + Namespace: "jumpstarter", + Client: "ci", + }}}) + if err != nil { + t.Fatal(err) + } + if resp.Accepted != 1 { + t.Fatalf("Accepted = %d, want 1", resp.Accepted) + } + if err := pusher.Flush(context.Background()); err != nil { + t.Fatal(err) + } + var payload lokiPushPayload + if err := json.Unmarshal(gotBody, &payload); err != nil { + t.Fatalf("body %s: %v", gotBody, err) + } + if payload.Streams[0].Stream["exporter"] != "real-exporter" { + t.Errorf("stream exporter = %q (must come from token, not the payload)", payload.Streams[0].Stream["exporter"]) + } +} + +func counterValue(t *testing.T, vec *prometheus.CounterVec, dest string) float64 { + t.Helper() + m, err := vec.GetMetricWithLabelValues(dest) + if err != nil { + t.Fatal(err) + } + var pb dto.Metric + if err := m.Write(&pb); err != nil { + t.Fatal(err) + } + return pb.GetCounter().GetValue() +} + +func TestLokiPush_InsecureSkipVerify(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(srv.Close) + + p, err := NewLokiPusher(LokiConfig{ + URL: srv.URL, + InsecureSkipVerify: true, + QueueDepth: 4, + }, testDroppedCounter()) + if err != nil { + t.Fatal(err) + } + p.Enqueue(logEntry("tls")) + if err := p.Flush(context.Background()); err != nil { + t.Fatalf("Flush with insecureSkipVerify: %v", err) + } +} diff --git a/controller/internal/service/telemetry_http.go b/controller/internal/service/telemetry_http.go index ecd403019..91357cf34 100644 --- a/controller/internal/service/telemetry_http.go +++ b/controller/internal/service/telemetry_http.go @@ -43,7 +43,12 @@ func (s *TelemetryService) initScrapeTimeouts() { Name: metricsParseErrorsMetric, Help: "Exporter MetricsStream snapshots omitted because OpenMetrics parse failed.", }, []string{labelExporter}) - s.metricsRegistry.MustRegister(s.scrapeTimeouts, s.parseErrors) + s.droppedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: droppedTotalMetric, + Help: "Log entries dropped due to Loki backpressure.", + }, []string{"destination"}) + s.metricsRegistry.MustRegister(s.scrapeTimeouts, s.parseErrors, s.droppedTotal) + s.droppedTotal.WithLabelValues(droppedDestination).Add(0) } func (s *TelemetryService) recordMetricsParseError(exporter string, err error) { @@ -105,9 +110,7 @@ func (s *TelemetryService) handleHealthz(w http.ResponseWriter, _ *http.Request) } // handleReadyz is 200 once the gRPC listener is bound. -// JEP DD-7 also gates on Loki reachability and a connected exporter; Loki is -// Phase 3 PR D, and requiring an exporter would keep an empty lab unready -// (Prometheus could not scrape jumpstarter_scrape_timeouts_total). +// Loki is optional (metrics-only is valid) and is not a readiness gate. func (s *TelemetryService) handleReadyz(w http.ResponseWriter, _ *http.Request) { if !s.grpcReady.Load() { http.Error(w, "grpc not ready\n", http.StatusServiceUnavailable) diff --git a/controller/internal/service/telemetry_service.go b/controller/internal/service/telemetry_service.go index b2be84936..05ecafaeb 100644 --- a/controller/internal/service/telemetry_service.go +++ b/controller/internal/service/telemetry_service.go @@ -59,7 +59,7 @@ var reservedExtraFieldKeys = map[string]struct{}{ } // TelemetryService receives structured log entries from exporters and clients, -// logs them via structured stdout, and will forward them to Loki in a future phase. +// logs them via structured stdout, and optionally forwards them to Loki. // // TLS: the server always uses TLS. When EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM // env vars point to certificate/key files (mounted by the operator from a Secret), @@ -92,18 +92,25 @@ type TelemetryService struct { // without a k8s client. Signer *oidc.Signer + // LokiConfig is used by Start to construct a LokiPusher when Loki is nil. + LokiConfig LokiConfig + + // Loki, when non-nil, forwards accepted PushLogs entries to Loki's HTTP API. + Loki *LokiPusher + stateMu sync.Mutex conns map[string]*metricsConn scrapeTimeouts prometheus.Counter parseErrors *prometheus.CounterVec + droppedTotal *prometheus.CounterVec metricsRegistry *prometheus.Registry metricsAddr string grpcReady atomic.Bool } -// PushLogs receives a batch of structured log entries and writes them via the -// controller-runtime logger (structured JSON to stdout). -// Future phase: forward to Loki push API. +// PushLogs receives a batch of structured log entries, writes them via the +// controller-runtime logger (structured JSON to stdout), and enqueues them for +// Loki when a pusher is configured. func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest) (*pb.PushLogsResponse, error) { id, err := s.authenticateExporter(ctx) if err != nil { @@ -136,56 +143,49 @@ func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest continue } + prepared := prepareLogEntry(id, entry) + // Always log the authenticated identity. After the mismatch checks // above, any non-empty entry fields already match the token; using // the token values makes the server the source of truth for Loki // stream labels even when the entry omitted them. kvs := []any{ - "component", entry.Component, + "component", prepared.Component, logFieldExporter, claimedName, "namespace", claimedNamespace, - "severity", entry.Severity, + "severity", prepared.Severity, } - if entry.Timestamp != nil { - kvs = append(kvs, "ts", entry.Timestamp.AsTime().Format(time.RFC3339Nano)) + if prepared.Timestamp != nil { + kvs = append(kvs, "ts", prepared.Timestamp.AsTime().Format(time.RFC3339Nano)) } - if entry.Lease != "" { - kvs = append(kvs, "lease", entry.Lease) + if prepared.Lease != "" { + kvs = append(kvs, "lease", prepared.Lease) } - if entry.Client != "" { - kvs = append(kvs, "client", entry.Client) + if prepared.Client != "" { + kvs = append(kvs, "client", prepared.Client) } - if entry.Operation != "" { - kvs = append(kvs, "operation", entry.Operation) + if prepared.Operation != "" { + kvs = append(kvs, "operation", prepared.Operation) } - if entry.Result != "" { - kvs = append(kvs, "result", entry.Result) + if prepared.Result != "" { + kvs = append(kvs, "result", prepared.Result) } - if entry.DriverType != "" { - kvs = append(kvs, "driver_type", entry.DriverType) + if prepared.DriverType != "" { + kvs = append(kvs, "driver_type", prepared.DriverType) } - // Enforce extra_fields limits and strip reserved keys so an exporter - // cannot shadow trusted fields in downstream log parsers. - count := 0 - for k, v := range entry.ExtraFields { - if count >= maxExtraFields { - break - } - if _, reserved := reservedExtraFieldKeys[k]; reserved { - continue - } - k = truncate(k, maxKeyLen) - v = truncate(v, maxValueLen) + for k, v := range prepared.ExtraFields { kvs = append(kvs, k, v) - count++ } - switch strings.ToLower(entry.Severity) { + switch strings.ToLower(prepared.Severity) { case "error", "critical": - logger.Error(nil, entry.Message, kvs...) + logger.Error(nil, prepared.Message, kvs...) default: - logger.Info(entry.Message, kvs...) + logger.Info(prepared.Message, kvs...) + } + if s.Loki != nil { + s.Loki.Enqueue(prepared) } accepted++ } @@ -196,6 +196,43 @@ func (s *TelemetryService) PushLogs(ctx context.Context, req *pb.PushLogsRequest }, nil } +// prepareLogEntry copies entry with identity overwritten from the token and +// extra_fields truncated / stripped of reserved keys. +func prepareLogEntry(id exporterIdentity, entry *pb.LogEntry) *pb.LogEntry { + out := &pb.LogEntry{ + Timestamp: entry.Timestamp, + Severity: entry.Severity, + Message: entry.Message, + Component: entry.Component, + Exporter: id.name, + Lease: entry.Lease, + Client: entry.Client, + Operation: entry.Operation, + Result: entry.Result, + DriverType: entry.DriverType, + Namespace: id.namespace, + } + if len(entry.ExtraFields) == 0 { + return out + } + extra := make(map[string]string, len(entry.ExtraFields)) + count := 0 + for k, v := range entry.ExtraFields { + if count >= maxExtraFields { + break + } + if _, reserved := reservedExtraFieldKeys[k]; reserved { + continue + } + extra[truncate(k, maxKeyLen)] = truncate(v, maxValueLen) + count++ + } + if len(extra) > 0 { + out.ExtraFields = extra + } + return out +} + // truncate returns s truncated to at most n bytes (rune-safe: truncates at rune boundary). func truncate(s string, n int) string { if len(s) <= n { @@ -270,6 +307,18 @@ func (s *TelemetryService) Start(ctx context.Context) error { s.initScrapeTimeouts() s.grpcReady.Store(true) + if s.Loki == nil { + pusher, lokiErr := NewLokiPusher(s.LokiConfig, s.droppedTotal) + if lokiErr != nil { + logger.Error(lokiErr, "Loki push disabled") + } else { + s.Loki = pusher + } + } + if s.Loki != nil { + go s.Loki.Run(ctx) + } + httpShutdown, err := s.startMetricsHTTP() if err != nil { s.grpcReady.Store(false) From b2abe3661704ddc4488b85e7030f42d285dcf8af Mon Sep 17 00:00:00 2001 From: Roddie Kieley Date: Wed, 2 Sep 2026 20:33:36 -0230 Subject: [PATCH 5/5] fix: incorrect apache license headers. Signed-off-by: Roddie Kieley --- .../internal/controller/jumpstarter/telemetry_loki_test.go | 2 +- controller/internal/service/loki_push_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go index 66cdc5018..65da7d4a8 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_loki_test.go @@ -7,7 +7,7 @@ You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - 10|Unless required by applicable law or agreed to in writing, software +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and diff --git a/controller/internal/service/loki_push_test.go b/controller/internal/service/loki_push_test.go index 0122aa9ee..966407fd6 100644 --- a/controller/internal/service/loki_push_test.go +++ b/controller/internal/service/loki_push_test.go @@ -7,7 +7,7 @@ You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - 10|Unless required by applicable law or agreed to in writing, software +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and