JEP-0013 - Phase 3 - client PushLogs

controller/Containerfile

@@ -48,11 +48,18 @@ RUN  --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1
      go build -a \
      -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \
      -o router ./cmd/router
+RUN  --mount=type=cache,target=/opt/app-root/src/go/pkg/mod,sharing=locked,uid=1001,gid=0 \
+     --mount=type=cache,target=/opt/app-root/src/.cache/go-build,sharing=locked,uid=1001,gid=0 \
+     CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \
+     go build -a \
+     -ldflags "-X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \
+     -o telemetry ./cmd/telemetry
 
 FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1786321990@sha256:7e7f79ab747bf2b452e3043dd89f388e92be4c7fdcc8b815b58adf6c99c39c95
 WORKDIR /
 COPY --from=builder /build/manager .
 COPY --from=builder /build/router  .
+COPY --from=builder /build/telemetry .
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

controller/Makefile

@@ -123,6 +123,7 @@ build-operator-ci:
 build: manifests generate fmt vet ## Build manager binary.
 	go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go
 	go build -ldflags "$(LDFLAGS)" -o bin/router ./cmd/router
+	go build -ldflags "$(LDFLAGS)" -o bin/telemetry ./cmd/telemetry
 	go build -ldflags "$(LDFLAGS)" -o bin/exporter-set-controller cmd/exporter-set-controller/main.go
 
 .PHONY: run
@@ -149,6 +150,7 @@ docker-build-ci: ## Build docker images from pre-compiled host binaries (fast CI
 	rm -rf bin/ci-stage && mkdir -p bin/ci-stage/controller bin/ci-stage/esc
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/manager cmd/main.go
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/router ./cmd/router
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/controller/telemetry ./cmd/telemetry
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags "$(LDFLAGS)" -o bin/ci-stage/esc/exporter-set-controller cmd/exporter-set-controller/main.go
 	$(CONTAINER_TOOL) build --build-arg BIN=manager -t $(IMG) -f Containerfile.prebuilt bin/ci-stage/controller
 	$(CONTAINER_TOOL) build --build-arg BIN=exporter-set-controller -t $(EXPORTER_SET_CONTROLLER_IMG) -f Containerfile.prebuilt bin/ci-stage/esc

controller/cmd/telemetry/main.go

@@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// jumpstarter-telemetry receives structured log entries from exporters and clients
-// via the PushLogs gRPC RPC and writes them to structured stdout for downstream
-// log shippers (Promtail, Grafana Alloy, Vector) to forward to Loki.
+// jumpstarter-telemetry reverse-scrapes exporter metrics via MetricsStream and
+// receives structured log entries via PushLogs. Logs are written to structured
+// stdout for downstream log shippers (Promtail, Grafana Alloy, Vector) and
+// optionally pushed to Loki's HTTP API when -loki-url is set.
 //
 // TLS: always enabled. Set EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM to file paths of
 // operator-mounted cert/key (e.g. from a cert-manager Secret); when absent a
@@ -30,16 +31,17 @@ limitations under the License.
 // certificate; the controller uses it to advertise the address to exporters via
 // GetServiceEndpoints. A mismatch causes TLS hostname verification failures.
 //
-// Future phases will add direct Loki push and MetricsStream for reverse-scrape
-// of exporter prometheus_client registries.
+// HTTP: GET /metrics, /healthz, and /readyz bind separately (default :8080).
 package main
 
 import (
 	"context"
 	"flag"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
+	"time"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -55,9 +57,48 @@ var (
 	buildDate = "unknown"
 )
 
+func splitCSV(s string) []string {
+	if s == "" {
+		return nil
+	}
+	parts := strings.Split(s, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
 func main() {
 	var bindAddr string
+	var metricsAddr string
+	var scrapeTimeout time.Duration
+	var driverTypeEnum string
+	var exemplarKeys string
+	var lokiURL string
+	var lokiQueueDepth int
+	var lokiInsecure bool
+	var lokiCAFile string
 	flag.StringVar(&bindAddr, "grpc-bind", ":9093", "TCP address to bind the gRPC server to")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080",
+		"TCP address for HTTP GET /metrics, /healthz, and /readyz. Use 0 to disable.")
+	flag.DurationVar(&scrapeTimeout, "scrape-timeout", 7*time.Second,
+		"Max wait for parallel MetricsStream scrape responses")
+	flag.StringVar(&driverTypeEnum, "driver-type-enum", strings.Join(service.DefaultDriverTypeEnum, ","),
+		"Comma-separated allowlist of driver_type values; others are remapped to other")
+	flag.StringVar(&exemplarKeys, "exemplar-keys", strings.Join(service.DefaultExemplarKeys, ","),
+		"Comma-separated allowlist of Prometheus exemplar keys")
+	flag.StringVar(&lokiURL, "loki-url", "",
+		"Loki HTTP push URL (optional; telemetry runs metrics-only when empty)")
+	flag.IntVar(&lokiQueueDepth, "loki-queue-depth", 10000,
+		"Ring buffer depth for Loki log push")
+	flag.BoolVar(&lokiInsecure, "loki-insecure-skip-verify", false,
+		"Disable TLS certificate verification for Loki (development/testing only)")
+	flag.StringVar(&lokiCAFile, "loki-ca-file", "",
+		"PEM CA bundle used to verify the Loki TLS endpoint")
 
 	opts := zap.Options{}
 	opts.BindFlags(flag.CommandLine)
@@ -71,6 +112,8 @@ func main() {
 		"gitCommit", gitCommit,
 		"buildDate", buildDate,
 		"bindAddr", bindAddr,
+		"metricsBindAddr", metricsAddr,
+		"scrapeTimeout", scrapeTimeout,
 	)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -87,8 +130,24 @@ func main() {
 	}
 
 	svc := &service.TelemetryService{
-		BindAddr: bindAddr,
-		Signer:   signer,
+		BindAddr:        bindAddr,
+		MetricsBindAddr: metricsAddr,
+		ScrapeTimeout:   scrapeTimeout,
+		DriverTypeEnum:  splitCSV(driverTypeEnum),
+		ExemplarKeys:    splitCSV(exemplarKeys),
+		Signer:          signer,
+		LokiConfig: service.LokiConfig{
+			URL:                lokiURL,
+			Username:           os.Getenv("LOKI_USERNAME"),
+			Password:           os.Getenv("LOKI_PASSWORD"),
+			Token:              os.Getenv("LOKI_TOKEN"),
+			CAFile:             lokiCAFile,
+			InsecureSkipVerify: lokiInsecure,
+			QueueDepth:         lokiQueueDepth,
+		},
+	}
+	if lokiURL != "" {
+		logger.Info("Loki HTTP push configured", "url", lokiURL, "queueDepth", lokiQueueDepth)
 	}
 
 	// Register signal handler before starting the service so no signal

controller/deploy/operator/api/v1alpha1/jumpstarter_types.go

@@ -312,6 +312,50 @@ type TelemetryConfig struct {
 	// gRPC configuration for the telemetry service.
 	// Use this to configure TLS when not using cert-manager.
 	GRPC TelemetryGRPCConfig `json:"grpc,omitempty"`
+
+	// Metrics configures reverse-scrape fan-out and Prometheus exposition
+	// (JEP-0013). ServiceMonitor fields are a later phase.
+	Metrics TelemetryMetricsConfig `json:"metrics,omitempty"`
+
+	// Loki configures optional HTTP push of ingested logs to a Loki-compatible
+	// endpoint. When url is empty, telemetry runs metrics-only.
+	Loki TelemetryLokiConfig `json:"loki,omitempty"`
+
+	// Backpressure configures the Loki log push ring buffer.
+	Backpressure TelemetryBackpressureConfig `json:"backpressure,omitempty"`
+}
+
+// TelemetryLokiConfig configures Loki HTTP push from the telemetry service.
+type TelemetryLokiConfig struct {
+	// Loki push endpoint (http:// or https://). Optional — telemetry can run
+	// metrics-only without Loki. grpc:// is reserved but not implemented yet.
+	URL string `json:"url,omitempty"`
+
+	// Secret with Loki credentials (username/password and/or token keys).
+	// See JEP-0013 DD-5: only the telemetry pod holds Loki credentials.
+	SecretRef string `json:"secretRef,omitempty"`
+
+	// TLS settings for the Loki endpoint.
+	TLS TelemetryLokiTLSConfig `json:"tls,omitempty"`
+}
+
+// TelemetryLokiTLSConfig configures TLS for the Loki push endpoint.
+type TelemetryLokiTLSConfig struct {
+	// Secret containing a CA bundle (ca.crt key) to trust for the Loki endpoint.
+	CASecretRef string `json:"caSecretRef,omitempty"`
+
+	// Disable TLS certificate verification (development/testing only).
+	// +kubebuilder:default=false
+	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
+}
+
+// TelemetryBackpressureConfig configures the Loki log push ring buffer.
+type TelemetryBackpressureConfig struct {
+	// Ring buffer depth for Loki log push. On overflow, dropped entries are
+	// replaced by a single drop-marker LogEntry.
+	// +kubebuilder:default=10000
+	// +kubebuilder:validation:Minimum=1
+	QueueDepth int32 `json:"queueDepth,omitempty"`
 }
 
 // TelemetryGRPCConfig defines gRPC configuration for the telemetry service.
@@ -325,6 +369,22 @@ type TelemetryGRPCConfig struct {
 	TLS TLSConfig `json:"tls,omitempty"`
 }
 
+// TelemetryMetricsConfig configures telemetry /metrics reverse-scrape behavior.
+type TelemetryMetricsConfig struct {
+	// Allowlist of keys to include in Prometheus exemplars. Unlisted keys are omitted.
+	// +kubebuilder:default={"client","lease_id"}
+	ExemplarKeys []string `json:"exemplarKeys,omitempty"`
+
+	// Allowed driver_type label values. Unlisted types are remapped to "other".
+	// +kubebuilder:default={"power","storage","network","serial","console","video","composite"}
+	DriverTypeEnum []string `json:"driverTypeEnum,omitempty"`
+
+	// Max wait for parallel exporter MetricsStream responses during a /metrics fan-out.
+	// Should be lower than the Prometheus scrape_timeout.
+	// +kubebuilder:default="7s"
+	ScrapeTimeout *metav1.Duration `json:"scrapeTimeout,omitempty"`
+}
+
 // TelemetryLoggingConfig configures the log push path to the telemetry service.
 type TelemetryLoggingConfig struct {
 	// Filter controls which log entries are forwarded to the telemetry service.