feat: make metrics and logs top-level verbs

README.md

@@ -548,13 +548,13 @@ spinloop fleet start gpu-box   # start one node's engine
 
 `dashboard` is the fleet you actually look at, and the board
 [at the top of this page](#2-on-every-machine-you-own) is a real one: one tile per
-node, repainted in place, showing the same numbers `fleet metrics` prints —
+node, repainted in place, showing the same numbers `metrics` prints —
 start a node with `s`, stop one with `x`, and a waking cloud machine shows its
 progress on its own tile; `a` lets you stop watching one that is still waking —
 it carries on in the cloud.
 Press `<enter>` on a tile for a full-screen view of that node — metrics, its
 engine log tailed live, and the keys that work there — `<esc>` to go back.
-`fleet metrics --watch` is the same board as a stream, for pipes.
+`metrics --watch` is the same board as a stream, for pipes.
 
 ```
 NODE     STATE         SERVING
@@ -625,11 +625,11 @@ while you are using it, and stops itself after a period of idleness.
 spinloop remote start --env dev-2 --print-env   # boot the instance, wait for the
                           # model to load, then print OPENAI_BASE_URL /
                           # OPENAI_API_KEY exports for eval
-spinloop remote status --env dev-2              # instance state, endpoint health,
+spinloop status --env <name> --env dev-2              # instance state, endpoint health,
                           # and when it last did any work
-spinloop remote metrics --env dev-2             # tokens, GPU, CPU and RAM — plus
+spinloop metrics --env <name> --env dev-2             # tokens, GPU, CPU and RAM — plus
                           # the same last-active
-spinloop remote logs --env dev-2                # what the engine (or the boot)
+spinloop logs --env <name> --env dev-2                # what the engine (or the boot)
                           # said, even after it's gone
 spinloop remote pause --env dev-2               # stop now, but keep it re-wakeable
 spinloop remote restart --env dev-2             # fresh engine, same address: stop

cmd/spinloop/commands.go

@@ -76,6 +76,8 @@ has been.`,
 		upCmd(),
 		statusCmd(),
 		dashboardCmd(),
+		metricsCmd(),
+		logsCmd(),
 		codeCmd(),
 		daemonCmd(),
 		gatewayCmd(),
@@ -349,6 +351,11 @@ var movedSubcommands = map[string]string{
 	"fleet harness":   "code --fleet <path>",
 	"fleet status":    "status",
 	"fleet dashboard": "dashboard",
+	"fleet metrics":   "metrics",
+	"fleet logs":      "logs",
+	"remote status":   "status --env <name>",
+	"remote metrics":  "metrics --env <name>",
+	"remote logs":     "logs --env <name>",
 }
 
 // fleetCmd builds the fleet parent and its subcommands. The parent does
@@ -370,8 +377,6 @@ error — only a problem with the fleet file itself fails a command.`,
 		RunE:          groupFallback,
 	}
 	fleet.AddCommand(
-		fleetMetricsCmd(),
-		fleetLogsCmd(),
 		fleetRouteCmd(),
 		fleetStartCmd(),
 		fleetStopCmd(),
@@ -404,9 +409,6 @@ names a file — falling back to the default environment. Each subcommand's
 		remotePauseCmd(),
 		remoteRestartCmd(),
 		remoteStopCmd(),
-		remoteStatusCmd(),
-		remoteMetricsCmd(),
-		remoteLogsCmd(),
 		remoteDeployCmd(),
 		remoteSeedCmd(),
 		remoteEnvCmd(),

cmd/spinloop/dashboard.go

@@ -11,7 +11,7 @@ import (
 )
 
 func dashboardCmd() *cobra.Command {
-	var path, envName string
+	var path, envName, spinloopPath string
 	c := &cobra.Command{
 		Use:   "dashboard",
 		Short: "watch the engines in an interactive tiled view",
@@ -39,12 +39,16 @@ metrics --watch instead.`,
 		SilenceUsage:  true,
 		RunE: func(c *cobra.Command, _ []string) error {
 			resolve(c)
+			if err := applyReadSpinloopEnv(spinloopPath); err != nil {
+				return err
+			}
 			return runFleetDashboard(fleetTarget{envName: envName, fleetPath: path})
 		},
 	}
 	fs := c.Flags()
 	fs.StringVarP(&path, "fleet", "f", "", fleetFileUsage)
 	fs.StringVar(&envName, "env", "", envFlagTargetUsage)
+	registerSpinloopEnvFlag(fs, &spinloopPath)
 	c.ValidArgsFunction = noPositionals
 	compRegister(c, "fleet", compFiles)
 	compRegister(c, "env", compEnvs)

cmd/spinloop/flagparse_test.go

@@ -28,7 +28,7 @@ func TestPflagParseForms(t *testing.T) {
 			"add":           func() error { return cmdAdd([]string{"-p", "ollama", "--nope"}) },
 			"apply":         func() error { return cmdApply([]string{"--nope"}) },
 			"serve":         func() error { return cmdServe([]string{"--nope"}) },
-			"fleet metrics": func() error { return cmdFleet([]string{"metrics", "--nope"}) },
+			"fleet metrics": func() error { return cmdMetrics([]string{"--nope"}) },
 			"remote start":  func() error { return cmdRemoteStart([]string{"--env", "default", "--nope"}) },
 			"daemon":        func() error { return cmdDaemon([]string{"--nope"}) },
 		} {
@@ -50,7 +50,7 @@ func TestPflagParseForms(t *testing.T) {
 		// takes them in place. An unknown flag raised *after* the positional is
 		// the proof that parsing continued past it.
 		for name, call := range map[string]func() error{
-			"fleet metrics": func() error { return cmdFleet([]string{"metrics", "someNode", "--nope"}) },
+			"fleet metrics": func() error { return cmdMetrics([]string{"someNode", "--nope"}) },
 			"remote env":    func() error { return cmdRemoteEnv([]string{"--env", "default", "somePath", "--nope"}) },
 		} {
 			if err := call(); err == nil || !strings.Contains(err.Error(), "unknown flag: --nope") {

cmd/spinloop/fleet.go

@@ -62,65 +62,32 @@ func fleetRow(r fleet.NodeResult) (state, serving string) {
 	}
 	// The shared facts come from the same source the remote status view reads,
 	// so the two cannot word or compute them differently.
+	// A node that runs on an instance reports its release outside the status
+	// reply, so the instance's answer fills what the reply left empty rather
+	// than overriding a daemon that carries its own.
+	version := r.Status.Version
+	if version == "" {
+		version = r.Instance.Version
+	}
 	f := statusFact{
 		State:         r.Status.State,
 		Model:         r.Status.Model,
 		Runner:        r.Status.Runner,
-		Version:       r.Status.Version,
+		Version:       version,
 		UptimeSeconds: r.Status.UptimeSeconds,
 		LastActiveAt:  r.Status.LastActiveAt,
 		IdleSeconds:   r.Status.IdleSeconds,
 		Ready:         r.Status.Ready,
+		Endpoint:      r.Instance.BaseURL,
+		RetainUntil:   r.Instance.RetainUntil,
 	}
 	return f.State, f.servingText()
 }
 
-// fleetMetricsCmd renders every node's engine and system metrics. --watch
-// redraws the whole fleet on an interval.
-func fleetMetricsCmd() *cobra.Command {
-	var (
-		path    string
-		envName string
-		format  string
-		watch   bool
-	)
-	c := &cobra.Command{
-		Use:           "metrics",
-		Short:         "sample every node's engine metrics",
-		Args:          cobra.ArbitraryArgs,
-		SilenceErrors: true,
-		SilenceUsage:  true,
-		RunE: func(c *cobra.Command, _ []string) error {
-			resolve(c)
-			if err := validateMetricsFormat(format); err != nil {
-				return err
-			}
-			cfg, err := resolveFleetTarget(fleetTarget{envName: envName, fleetPath: path})
-			if err != nil {
-				return err
-			}
-			if watch {
-				return runFleetMetricsWatch(cfg, format)
-			}
-			results := cfg.FanOut(context.Background(), fleet.MetricsCall)
-			return renderFleetMetrics(os.Stdout, results, format)
-		},
-	}
-	fs := c.Flags()
-	fs.StringVarP(&path, "fleet", "f", "", fleetFileUsage)
-	fs.StringVar(&envName, "env", "", envFlagTargetUsage)
-	fs.StringVar(&format, "format", "gauge", "output format: gauge (default), bar, table or json")
-	fs.BoolVarP(&watch, "watch", "w", false, "redraw the fleet every 60 seconds")
-	c.ValidArgsFunction = noPositionals
-	compRegister(c, "fleet", compFiles)
-	compRegister(c, "env", compEnvs)
-	return c
-}
-
 // runFleetMetricsWatch redraws the fleet until interrupted. Each refresh is
 // rendered into a buffer first, so the screen is cleared and rewritten in one
 // go — a slow node delays a refresh but never tears the display.
-func runFleetMetricsWatch(cfg *fleet.Config, format string) error {
+func runFleetMetricsWatch(cfg *fleet.Config, format string, call fleet.Call) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -135,10 +102,19 @@ func runFleetMetricsWatch(cfg *fleet.Config, format string) error {
 	first := true
 	for {
 		var buf strings.Builder
-		results := cfg.FanOut(ctx, fleet.MetricsCall)
+		results := cfg.FanOut(ctx, call)
 		if ctx.Err() != nil {
 			return nil
 		}
+		// A single-node target (an --env watch, or a fleet of one) with
+		// nothing else to show for the poll ends the watch rather than
+		// redrawing the same failure forever: this is the one node the
+		// caller asked about, and it could not be read. A multi-node fleet
+		// keeps drawing through a bad node so the rest of the fleet stays
+		// visible.
+		if len(results) == 1 && !results[0].OK() {
+			return fmt.Errorf("%s: %s", results[0].Name, results[0].Detail())
+		}
 		if err := renderFleetMetrics(&buf, results, format); err != nil {
 			return err
 		}
@@ -173,16 +149,19 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string)
 			continue
 		}
 		stats := r.Metrics
-		fmt.Fprintf(w, "%s  %s", r.Name, stats.State)
-		if stats.ModelID != "" {
-			fmt.Fprintf(w, "  %s", stats.ModelID)
-		}
-		fmt.Fprintln(w)
+		renderMetricsHeader(w, r, format)
 		// Before the continue, for the same reason the remote formats show it
 		// before theirs: a node whose engine has stopped still has a useful
 		// answer to "when did it last do anything?" — and, for a retained
 		// remote environment, "how long is it kept?".
-		renderActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds, stats.RetainUntil, now)
+		// The table spells its facts as key-value lines, so its active line is
+		// spelled that way too; the compact formats indent theirs under the
+		// header.
+		if format == "table" {
+			renderActiveKeyValue(w, stats.LastActiveAt, stats.IdleSeconds, retainUntilOf(r), now)
+		} else {
+			renderActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds, retainUntilOf(r), now)
+		}
 		switch format {
 		case "bar":
 			// No state gate, for the same reason the remote bar format has
@@ -205,11 +184,71 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string)
 			renderGPUTable(w, stats.GPUs)
 			renderCPUMemTable(w, stats.CPU, stats.Memory)
 		}
+		renderCost(w, r.Cost)
 		renderCollectionErrors(os.Stderr, stats.Errors)
 	}
 	return nil
 }
 
+// renderMetricsHeader draws the lines a node's figures open with. The table
+// format spells each fact on its own line, as the environment-only format did;
+// the compact formats put them on one line, as its bar and gauge did. Between
+// them they carry every fact those formats carried: what the node is, what it
+// runs on, what it serves, the release on it, and how long it has been up.
+func renderMetricsHeader(w io.Writer, r fleet.NodeResult, format string) {
+	stats := r.Metrics
+	if format == "table" {
+		fmt.Fprintf(w, "node:         %s\n", r.Name)
+		fmt.Fprintf(w, "state:        %s\n", stats.State)
+		for _, line := range []struct{ label, value string }{
+			{"instance", r.Instance.ID},
+			{"instanceType", r.Instance.Type},
+			{"runner", stats.Runner},
+			{"model", stats.ModelID},
+			{"version", r.Instance.Version},
+			{"endpoint", r.Instance.BaseURL},
+		} {
+			if line.value != "" {
+				fmt.Fprintf(w, "%-13s %s\n", line.label+":", line.value)
+			}
+		}
+		if stats.UptimeSeconds > 0 {
+			fmt.Fprintf(w, "%-13s %s\n", "uptime:", formatDuration(stats.UptimeSeconds))
+		}
+		return
+	}
+	fmt.Fprintf(w, "%s  %s", r.Name, stats.State)
+	for _, v := range []string{r.Instance.Type, stats.ModelID, r.Instance.Version} {
+		if v != "" {
+			fmt.Fprintf(w, "  %s", v)
+		}
+	}
+	if stats.UptimeSeconds > 0 {
+		fmt.Fprintf(w, "  (up %s)", formatDuration(stats.UptimeSeconds))
+	}
+	fmt.Fprintln(w)
+}
+
+// retainUntilOf is a node's retention deadline from whichever answer carried
+// it: the shared stats where a reading has been taken, the instance the node
+// describes otherwise. One fact, two replies, so a caller never has to pick.
+func retainUntilOf(r fleet.NodeResult) string {
+	if r.Metrics.RetainUntil != "" {
+		return r.Metrics.RetainUntil
+	}
+	return r.Instance.RetainUntil
+}
+
+// renderCost draws what a node has cost, where the caller asked and the node
+// could be priced. A node with no figure draws nothing: a zero would claim it
+// cost nothing.
+func renderCost(w io.Writer, c fleet.Cost) {
+	if !c.Reported() {
+		return
+	}
+	fmt.Fprintf(w, "  cost so far:  $%.2f (%.4f/hr)\n", c.SoFar, c.PerHour)
+}
+
 // fleetNodeJSON is one node in the JSON output: its metrics when it answered,
 // its outcome and reason when it did not — so a consumer sees the whole fleet
 // rather than silently missing the nodes that were down.
@@ -218,16 +257,35 @@ type fleetNodeJSON struct {
 	Outcome string `json:"outcome"`
 	Error   string `json:"error,omitempty"`
 	Metrics *any   `json:"metrics,omitempty"`
+	// The instance facts and the cost are what the node's kind could say
+	// beyond the shared engine figures. Omitted for a node that said none,
+	// which is every node that is a machine rather than an instance.
+	Instance     string   `json:"instance,omitempty"`
+	InstanceType string   `json:"instanceType,omitempty"`
+	Version      string   `json:"version,omitempty"`
+	BaseURL      string   `json:"baseUrl,omitempty"`
+	RetainUntil  string   `json:"retainUntil,omitempty"`
+	Cost         *float64 `json:"cost,omitempty"`
+	CostPerHour  *float64 `json:"costPerHour,omitempty"`
 }
 
 func renderFleetMetricsJSON(w io.Writer, results []fleet.NodeResult) error {
 	out := make([]fleetNodeJSON, 0, len(results))
 	for _, r := range results {
-		entry := fleetNodeJSON{Node: r.Name, Outcome: string(r.Outcome), Error: r.Detail()}
+		entry := fleetNodeJSON{
+			Node: r.Name, Outcome: string(r.Outcome), Error: r.Detail(),
+			Instance: r.Instance.ID, InstanceType: r.Instance.Type,
+			Version: r.Instance.Version, BaseURL: r.Instance.BaseURL,
+			RetainUntil: r.Instance.RetainUntil,
+		}
 		if r.OK() {
 			var m any = r.Metrics
 			entry.Metrics = &m
 		}
+		if r.Cost.Reported() {
+			soFar, perHour := r.Cost.SoFar, r.Cost.PerHour
+			entry.Cost, entry.CostPerHour = &soFar, &perHour
+		}
 		out = append(out, entry)
 	}
 	data, err := json.MarshalIndent(out, "", "  ")