Add Prometheus metrics endpoint for browser VM observability

server/README.md

@@ -51,6 +51,7 @@ Configure the server using environment variables:
 | Variable       | Default   | Description                                 |
 | -------------- | --------- | ------------------------------------------- |
 | `PORT`         | `10001`   | HTTP server port                            |
+| `METRICS_PORT` | `10002`   | Prometheus metrics port (`GET /metrics`)    |
 | `FRAME_RATE`   | `10`      | Default recording framerate (fps)           |
 | `DISPLAY_NUM`  | `1`       | Display/screen number to capture            |
 | `MAX_SIZE_MB`  | `500`     | Default maximum file size (MB)              |

server/cmd/api/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/kernel/kernel-images/server/lib/devtoolsproxy"
 	"github.com/kernel/kernel-images/server/lib/events"
 	"github.com/kernel/kernel-images/server/lib/logger"
+	"github.com/kernel/kernel-images/server/lib/metrics"
 	"github.com/kernel/kernel-images/server/lib/nekoclient"
 	oapi "github.com/kernel/kernel-images/server/lib/oapi"
 	"github.com/kernel/kernel-images/server/lib/recorder"
@@ -257,6 +258,22 @@ func main() {
 		Handler: rChromeDriver,
 	}
 
+	// Prometheus metrics for external collection. Served on its own
+	// listener, deliberately without the scale-to-zero middleware (periodic
+	// scrapes must not count as session activity) and without per-request
+	// logging (scrapes would drown the logs).
+	rMetrics := chi.NewRouter()
+	rMetrics.Use(chiMiddleware.Recoverer)
+	rMetrics.Method(http.MethodGet, "/metrics", metrics.Handler(slogger,
+		metrics.NewChromeCollector(upstreamMgr),
+		metrics.NewGPUCollector(),
+		metrics.NewSystemCollector(),
+	))
+	srvMetrics := &http.Server{
+		Addr:    fmt.Sprintf("0.0.0.0:%d", config.MetricsPort),
+		Handler: rMetrics,
+	}
+
 	go func() {
 		slogger.Info("http server starting", "addr", srv.Addr)
 		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
@@ -281,6 +298,14 @@ func main() {
 		}
 	}()
 
+	go func() {
+		slogger.Info("metrics server starting", "addr", srvMetrics.Addr)
+		if err := srvMetrics.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			slogger.Error("metrics server failed", "err", err)
+			stop()
+		}
+	}()
+
 	// graceful shutdown
 	<-ctx.Done()
 	slogger.Info("shutdown signal received")
@@ -308,6 +333,9 @@ func main() {
 	g.Go(func() error {
 		return srvChromeDriver.Shutdown(shutdownCtx)
 	})
+	g.Go(func() error {
+		return srvMetrics.Shutdown(shutdownCtx)
+	})
 
 	if err := g.Wait(); err != nil {
 		slogger.Error("server failed to shutdown", "err", err)

server/cmd/config/config.go

@@ -13,6 +13,11 @@ type Config struct {
 	// Server configuration
 	Port int `envconfig:"PORT" default:"10001"`
 
+	// Port for the Prometheus metrics endpoint. Served on a separate
+	// listener so scrapes bypass the scale-to-zero middleware and the
+	// external API surface.
+	MetricsPort int `envconfig:"METRICS_PORT" default:"10002"`
+
 	// Recording configuration
 	FrameRate   int    `envconfig:"FRAME_RATE" default:"10"`
 	DisplayNum  int    `envconfig:"DISPLAY_NUM" default:"1"`
@@ -57,6 +62,7 @@ func (c *Config) LogValue() slog.Value {
 	}
 	return slog.GroupValue(
 		slog.Int("port", c.Port),
+		slog.Int("metrics_port", c.MetricsPort),
 		slog.Int("frame_rate", c.FrameRate),
 		slog.Int("display_num", c.DisplayNum),
 		slog.Int("max_size_mb", c.MaxSizeInMB),

server/cmd/config/config_test.go

@@ -19,6 +19,7 @@ func TestLoad(t *testing.T) {
 			env:  map[string]string{},
 			wantCfg: &Config{
 				Port:                     10001,
+				MetricsPort:              10002,
 				FrameRate:                10,
 				DisplayNum:               1,
 				MaxSizeInMB:              500,
@@ -37,6 +38,7 @@ func TestLoad(t *testing.T) {
 			name: "custom valid env",
 			env: map[string]string{
 				"PORT":                       "12345",
+				"METRICS_PORT":               "12346",
 				"FRAME_RATE":                 "20",
 				"DISPLAY_NUM":                "2",
 				"MAX_SIZE_MB":                "250",
@@ -51,6 +53,7 @@ func TestLoad(t *testing.T) {
 			},
 			wantCfg: &Config{
 				Port:                     12345,
+				MetricsPort:              12346,
 				FrameRate:                20,
 				DisplayNum:               2,
 				MaxSizeInMB:              250,
@@ -73,6 +76,7 @@ func TestLoad(t *testing.T) {
 			},
 			wantCfg: &Config{
 				Port:                     10001,
+				MetricsPort:              10002,
 				FrameRate:                10,
 				DisplayNum:               1,
 				MaxSizeInMB:              500,

server/lib/cdpclient/cdpclient.go

@@ -133,6 +133,64 @@ func (c *Client) GetBrowserVersion(ctx context.Context) (*BrowserVersion, error)
 	return &v, nil
 }
 
+// Histogram is a snapshot of a Chrome UMA histogram as returned by
+// Browser.getHistograms. Values are cumulative since browser start and the
+// units follow the UMA definition of the histogram (PageLoad timings are
+// milliseconds). Only buckets with at least one sample are present.
+type Histogram struct {
+	Name    string            `json:"name"`
+	Sum     int64             `json:"sum"`
+	Count   int64             `json:"count"`
+	Buckets []HistogramBucket `json:"buckets"`
+}
+
+// HistogramBucket is a [Low, High) bucket of a Histogram.
+type HistogramBucket struct {
+	Low   int64 `json:"low"`
+	High  int64 `json:"high"`
+	Count int64 `json:"count"`
+}
+
+// GetHistograms sends Browser.getHistograms, a browser-level command that
+// reads Chrome's in-memory UMA histograms without attaching to any page.
+// query is a substring filter on the histogram name; empty returns all.
+func (c *Client) GetHistograms(ctx context.Context, query string) ([]Histogram, error) {
+	raw, err := c.send(ctx, "Browser.getHistograms", map[string]any{"query": query}, "")
+	if err != nil {
+		return nil, fmt.Errorf("Browser.getHistograms: %w", err)
+	}
+	var result struct {
+		Histograms []Histogram `json:"histograms"`
+	}
+	if err := json.Unmarshal(raw, &result); err != nil {
+		return nil, fmt.Errorf("unmarshal Browser.getHistograms: %w", err)
+	}
+	return result.Histograms, nil
+}
+
+// CountPageTargets returns the number of open page targets.
+func (c *Client) CountPageTargets(ctx context.Context) (int, error) {
+	targetsResult, err := c.send(ctx, "Target.getTargets", nil, "")
+	if err != nil {
+		return 0, fmt.Errorf("Target.getTargets: %w", err)
+	}
+	var targets struct {
+		TargetInfos []struct {
+			Type string `json:"type"`
+		} `json:"targetInfos"`
+	}
+	if err := json.Unmarshal(targetsResult, &targets); err != nil {
+		return 0, fmt.Errorf("unmarshal targets: %w", err)
+	}
+	n := 0
+	for _, t := range targets.TargetInfos {
+		if t.Type == "page" {
+			n++
+		}
+	}
+	return n, nil
+}
+
 // DispatchStartURL closes extra page targets and dispatches a navigation on the
 // first page target. It does not wait for lifecycle events; Chrome owns the
 // eventual navigation result.