From 1527f54af47b03e83347351d3f115d5eccdac796 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:10:34 +0000 Subject: [PATCH 1/3] Add Prometheus metrics endpoint for browser VM observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve GET /metrics on a dedicated port (default 10002) exposing: - Chrome page-load performance (FCP, LCP, DOMContentLoaded, load event, parse start, INP, CLS, page CPU) read browser-level via CDP Browser.getHistograms — no page attach, no script injection. - GPU presence, vGPU license state and lease expiry, utilization, and framebuffer memory from nvidia-smi (kernel_gpu_present 0 on non-GPU VMs). - VM resource metrics (CPU, memory, load, disk, network, uptime) and Chromium process count/RSS. The listener deliberately skips the scale-to-zero middleware so periodic scrapes don't count as session activity, and stays out of the OpenAPI spec like the other internal endpoints. Co-Authored-By: Claude Opus 4.7 --- server/README.md | 1 + server/cmd/api/main.go | 28 ++++ server/cmd/config/config.go | 6 + server/cmd/config/config_test.go | 4 + server/lib/cdpclient/cdpclient.go | 58 ++++++++ server/lib/metrics/chrome.go | 131 ++++++++++++++++++ server/lib/metrics/chrome_test.go | 113 ++++++++++++++++ server/lib/metrics/gpu.go | 197 +++++++++++++++++++++++++++ server/lib/metrics/gpu_test.go | 104 +++++++++++++++ server/lib/metrics/metrics.go | 98 ++++++++++++++ server/lib/metrics/metrics_test.go | 60 +++++++++ server/lib/metrics/system.go | 205 +++++++++++++++++++++++++++++ server/lib/metrics/system_test.go | 66 ++++++++++ 13 files changed, 1071 insertions(+) create mode 100644 server/lib/metrics/chrome.go create mode 100644 server/lib/metrics/chrome_test.go create mode 100644 server/lib/metrics/gpu.go create mode 100644 server/lib/metrics/gpu_test.go create mode 100644 server/lib/metrics/metrics.go create mode 100644 server/lib/metrics/metrics_test.go create mode 100644 server/lib/metrics/system.go create mode 100644 server/lib/metrics/system_test.go diff --git a/server/README.md b/server/README.md index 9eef65d7..cc80cb41 100644 --- a/server/README.md +++ b/server/README.md @@ -51,6 +51,7 @@ Configure the server using environment variables: | Variable | Default | Description | | -------------- | --------- | ------------------------------------------- | | `PORT` | `10001` | HTTP server port | +| `METRICS_PORT` | `10002` | Prometheus metrics port (`GET /metrics`) | | `FRAME_RATE` | `10` | Default recording framerate (fps) | | `DISPLAY_NUM` | `1` | Display/screen number to capture | | `MAX_SIZE_MB` | `500` | Default maximum file size (MB) | diff --git a/server/cmd/api/main.go b/server/cmd/api/main.go index 3a748024..babb8f40 100644 --- a/server/cmd/api/main.go +++ b/server/cmd/api/main.go @@ -27,6 +27,7 @@ import ( "github.com/kernel/kernel-images/server/lib/devtoolsproxy" "github.com/kernel/kernel-images/server/lib/events" "github.com/kernel/kernel-images/server/lib/logger" + "github.com/kernel/kernel-images/server/lib/metrics" "github.com/kernel/kernel-images/server/lib/nekoclient" oapi "github.com/kernel/kernel-images/server/lib/oapi" "github.com/kernel/kernel-images/server/lib/recorder" @@ -257,6 +258,22 @@ func main() { Handler: rChromeDriver, } + // Prometheus metrics for external collection. Served on its own + // listener, deliberately without the scale-to-zero middleware (periodic + // scrapes must not count as session activity) and without per-request + // logging (scrapes would drown the logs). + rMetrics := chi.NewRouter() + rMetrics.Use(chiMiddleware.Recoverer) + rMetrics.Method(http.MethodGet, "/metrics", metrics.Handler(slogger, + metrics.NewChromeCollector(upstreamMgr), + metrics.NewGPUCollector(), + metrics.NewSystemCollector(), + )) + srvMetrics := &http.Server{ + Addr: fmt.Sprintf("0.0.0.0:%d", config.MetricsPort), + Handler: rMetrics, + } + go func() { slogger.Info("http server starting", "addr", srv.Addr) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -281,6 +298,14 @@ func main() { } }() + go func() { + slogger.Info("metrics server starting", "addr", srvMetrics.Addr) + if err := srvMetrics.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slogger.Error("metrics server failed", "err", err) + stop() + } + }() + // graceful shutdown <-ctx.Done() slogger.Info("shutdown signal received") @@ -308,6 +333,9 @@ func main() { g.Go(func() error { return srvChromeDriver.Shutdown(shutdownCtx) }) + g.Go(func() error { + return srvMetrics.Shutdown(shutdownCtx) + }) if err := g.Wait(); err != nil { slogger.Error("server failed to shutdown", "err", err) diff --git a/server/cmd/config/config.go b/server/cmd/config/config.go index aae38b9e..290fbed9 100644 --- a/server/cmd/config/config.go +++ b/server/cmd/config/config.go @@ -13,6 +13,11 @@ type Config struct { // Server configuration Port int `envconfig:"PORT" default:"10001"` + // Port for the Prometheus metrics endpoint. Served on a separate + // listener so scrapes bypass the scale-to-zero middleware and the + // external API surface. + MetricsPort int `envconfig:"METRICS_PORT" default:"10002"` + // Recording configuration FrameRate int `envconfig:"FRAME_RATE" default:"10"` DisplayNum int `envconfig:"DISPLAY_NUM" default:"1"` @@ -57,6 +62,7 @@ func (c *Config) LogValue() slog.Value { } return slog.GroupValue( slog.Int("port", c.Port), + slog.Int("metrics_port", c.MetricsPort), slog.Int("frame_rate", c.FrameRate), slog.Int("display_num", c.DisplayNum), slog.Int("max_size_mb", c.MaxSizeInMB), diff --git a/server/cmd/config/config_test.go b/server/cmd/config/config_test.go index 27dd8b3d..ffda27c9 100644 --- a/server/cmd/config/config_test.go +++ b/server/cmd/config/config_test.go @@ -19,6 +19,7 @@ func TestLoad(t *testing.T) { env: map[string]string{}, wantCfg: &Config{ Port: 10001, + MetricsPort: 10002, FrameRate: 10, DisplayNum: 1, MaxSizeInMB: 500, @@ -37,6 +38,7 @@ func TestLoad(t *testing.T) { name: "custom valid env", env: map[string]string{ "PORT": "12345", + "METRICS_PORT": "12346", "FRAME_RATE": "20", "DISPLAY_NUM": "2", "MAX_SIZE_MB": "250", @@ -51,6 +53,7 @@ func TestLoad(t *testing.T) { }, wantCfg: &Config{ Port: 12345, + MetricsPort: 12346, FrameRate: 20, DisplayNum: 2, MaxSizeInMB: 250, @@ -73,6 +76,7 @@ func TestLoad(t *testing.T) { }, wantCfg: &Config{ Port: 10001, + MetricsPort: 10002, FrameRate: 10, DisplayNum: 1, MaxSizeInMB: 500, diff --git a/server/lib/cdpclient/cdpclient.go b/server/lib/cdpclient/cdpclient.go index 092ce69a..43313555 100644 --- a/server/lib/cdpclient/cdpclient.go +++ b/server/lib/cdpclient/cdpclient.go @@ -133,6 +133,64 @@ func (c *Client) GetBrowserVersion(ctx context.Context) (*BrowserVersion, error) return &v, nil } +// Histogram is a snapshot of a Chrome UMA histogram as returned by +// Browser.getHistograms. Values are cumulative since browser start and the +// units follow the UMA definition of the histogram (PageLoad timings are +// milliseconds). Only buckets with at least one sample are present. +type Histogram struct { + Name string `json:"name"` + Sum int64 `json:"sum"` + Count int64 `json:"count"` + Buckets []HistogramBucket `json:"buckets"` +} + +// HistogramBucket is a [Low, High) bucket of a Histogram. +type HistogramBucket struct { + Low int64 `json:"low"` + High int64 `json:"high"` + Count int64 `json:"count"` +} + +// GetHistograms sends Browser.getHistograms, a browser-level command that +// reads Chrome's in-memory UMA histograms without attaching to any page. +// query is a substring filter on the histogram name; empty returns all. +func (c *Client) GetHistograms(ctx context.Context, query string) ([]Histogram, error) { + raw, err := c.send(ctx, "Browser.getHistograms", map[string]any{"query": query}, "") + if err != nil { + return nil, fmt.Errorf("Browser.getHistograms: %w", err) + } + var result struct { + Histograms []Histogram `json:"histograms"` + } + if err := json.Unmarshal(raw, &result); err != nil { + return nil, fmt.Errorf("unmarshal Browser.getHistograms: %w", err) + } + return result.Histograms, nil +} + +// CountPageTargets returns the number of open page targets. +func (c *Client) CountPageTargets(ctx context.Context) (int, error) { + targetsResult, err := c.send(ctx, "Target.getTargets", nil, "") + if err != nil { + return 0, fmt.Errorf("Target.getTargets: %w", err) + } + var targets struct { + TargetInfos []struct { + Type string `json:"type"` + } `json:"targetInfos"` + } + if err := json.Unmarshal(targetsResult, &targets); err != nil { + return 0, fmt.Errorf("unmarshal targets: %w", err) + } + n := 0 + for _, t := range targets.TargetInfos { + if t.Type == "page" { + n++ + } + } + return n, nil +} + // DispatchStartURL closes extra page targets and dispatches a navigation on the // first page target. It does not wait for lifecycle events; Chrome owns the // eventual navigation result. diff --git a/server/lib/metrics/chrome.go b/server/lib/metrics/chrome.go new file mode 100644 index 00000000..e1ef1cc8 --- /dev/null +++ b/server/lib/metrics/chrome.go @@ -0,0 +1,131 @@ +package metrics + +import ( + "context" + "errors" + "math" + "strconv" + + "github.com/kernel/kernel-images/server/lib/cdpclient" +) + +// DefaultUMAHistograms is the curated set of Chrome UMA histograms exposed +// on the metrics endpoint. All are recorded per page load with millisecond +// units unless noted. Histograms with no samples yet are simply absent from +// the output. +var DefaultUMAHistograms = []string{ + "PageLoad.PaintTiming.NavigationToFirstContentfulPaint", + "PageLoad.PaintTiming.NavigationToLargestContentfulPaint2", + "PageLoad.DocumentTiming.NavigationToDOMContentLoadedEventFired", + "PageLoad.DocumentTiming.NavigationToLoadEventFired", + "PageLoad.ParseTiming.NavigationToParseStart", + // INP: worst-case interaction latency, high percentile per page load. + "PageLoad.InteractiveTiming.UserInteractionLatency.HighPercentile2.MaxEventDuration", + // CLS: unitless layout-shift score, not milliseconds. + "PageLoad.LayoutInstability.MaxCumulativeShiftScore.SessionWindow.Gap1000ms.Max5000ms2", + // CPU milliseconds consumed by the page across its lifetime. + "PageLoad.Cpu.TotalUsage", +} + +// DevToolsUpstream reports the current browser-level DevTools WebSocket URL. +// Satisfied by devtoolsproxy.UpstreamManager. +type DevToolsUpstream interface { + Current() string +} + +// ChromeCollector reads browser-wide metrics over CDP: UMA histograms via +// Browser.getHistograms plus the browser version and open page count. All +// commands are browser-level — nothing attaches to pages or injects into +// sessions, so scrapes are invisible to automation running in the browser. +type ChromeCollector struct { + upstream DevToolsUpstream + histograms []string +} + +func NewChromeCollector(upstream DevToolsUpstream) *ChromeCollector { + return &ChromeCollector{upstream: upstream, histograms: DefaultUMAHistograms} +} + +func (c *ChromeCollector) Name() string { return "chrome" } + +func (c *ChromeCollector) Collect(ctx context.Context, w *Writer) error { + w.Metric("kernel_chromium_up", "Whether the Chromium DevTools endpoint is reachable and responsive.", "gauge") + + client, version, err := c.dial(ctx) + if err != nil { + w.Sample("kernel_chromium_up", nil, 0) + return err + } + defer client.Close() + w.Sample("kernel_chromium_up", nil, 1) + + w.Metric("kernel_chromium_info", "Chromium build info; value is always 1.", "gauge") + w.Sample("kernel_chromium_info", []Label{{"product", version.Product}}, 1) + + if pages, err := client.CountPageTargets(ctx); err == nil { + w.Metric("kernel_chromium_pages", "Number of open page targets (tabs).", "gauge") + w.Sample("kernel_chromium_pages", nil, float64(pages)) + } + + w.Metric("kernel_chromium_uma", + "Chrome UMA histogram, cumulative since browser start. Units follow the UMA definition; PageLoad timings are milliseconds.", + "histogram") + for _, name := range c.histograms { + matches, err := client.GetHistograms(ctx, name) + if err != nil { + return err + } + for _, h := range matches { + // The query is a substring filter, so it also returns + // suffixed variants of the requested histogram. + if h.Name == name { + writeUMAHistogram(w, h) + } + } + } + return nil +} + +func (c *ChromeCollector) dial(ctx context.Context) (*cdpclient.Client, *cdpclient.BrowserVersion, error) { + url := c.upstream.Current() + if url == "" { + return nil, nil, errors.New("devtools upstream not available") + } + client, err := cdpclient.Dial(ctx, url) + if err != nil { + return nil, nil, err + } + version, err := client.GetBrowserVersion(ctx) + if err != nil { + client.Close() + return nil, nil, err + } + return client, version, nil +} + +// writeUMAHistogram emits one UMA histogram as a Prometheus histogram with a +// "histogram" label carrying the UMA name. Chrome reports sparse, disjoint +// [low, high) buckets; the upper bound maps to a cumulative le bound. The +// overflow bucket (high = MaxInt32) folds into +Inf. +func writeUMAHistogram(w *Writer, h cdpclient.Histogram) { + labels := func(le string) []Label { + return []Label{{"histogram", h.Name}, {"le", le}} + } + count := h.Count + if count == 0 { + for _, b := range h.Buckets { + count += b.Count + } + } + cumulative := int64(0) + for _, b := range h.Buckets { + if b.High >= math.MaxInt32 { + continue + } + cumulative += b.Count + w.Sample("kernel_chromium_uma_bucket", labels(strconv.FormatInt(b.High, 10)), float64(cumulative)) + } + w.Sample("kernel_chromium_uma_bucket", labels("+Inf"), float64(count)) + w.Sample("kernel_chromium_uma_sum", []Label{{"histogram", h.Name}}, float64(h.Sum)) + w.Sample("kernel_chromium_uma_count", []Label{{"histogram", h.Name}}, float64(count)) +} diff --git a/server/lib/metrics/chrome_test.go b/server/lib/metrics/chrome_test.go new file mode 100644 index 00000000..c33ae743 --- /dev/null +++ b/server/lib/metrics/chrome_test.go @@ -0,0 +1,113 @@ +package metrics + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/coder/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type staticUpstream string + +func (s staticUpstream) Current() string { return string(s) } + +// fakeCDP answers the browser-level commands the chrome collector issues. +func fakeCDP(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{OriginPatterns: []string{"*"}}) + require.NoError(t, err) + defer conn.Close(websocket.StatusNormalClosure, "") + + ctx := r.Context() + for { + _, msg, err := conn.Read(ctx) + if err != nil { + return + } + var req struct { + ID int64 `json:"id"` + Method string `json:"method"` + Params struct { + Query string `json:"query"` + } `json:"params"` + } + require.NoError(t, json.Unmarshal(msg, &req)) + + var result any + switch req.Method { + case "Browser.getVersion": + result = map[string]any{"product": "Chrome/145.0.7632.75"} + case "Target.getTargets": + result = map[string]any{"targetInfos": []map[string]any{ + {"targetId": "A", "type": "page"}, + {"targetId": "B", "type": "page"}, + {"targetId": "C", "type": "service_worker"}, + }} + case "Browser.getHistograms": + histograms := []map[string]any{} + if strings.Contains("PageLoad.PaintTiming.NavigationToFirstContentfulPaint", req.Params.Query) { + histograms = append(histograms, + map[string]any{ + "name": "PageLoad.PaintTiming.NavigationToFirstContentfulPaint", "sum": 21921, "count": 4, + "buckets": []map[string]any{ + {"low": 606, "high": 638, "count": 2}, + {"low": 819, "high": 862, "count": 1}, + {"low": 10000, "high": 2147483647, "count": 1}, + }, + }, + // Suffixed variant matched by the substring query; + // must be filtered out by the collector. + map[string]any{ + "name": "PageLoad.PaintTiming.NavigationToFirstContentfulPaint.UserInitiated", "sum": 1, "count": 1, + "buckets": []map[string]any{{"low": 0, "high": 2, "count": 1}}, + }, + ) + } + result = map[string]any{"histograms": histograms} + default: + result = map[string]any{} + } + resp, err := json.Marshal(map[string]any{"id": req.ID, "result": result}) + require.NoError(t, err) + require.NoError(t, conn.Write(ctx, websocket.MessageText, resp)) + } + })) +} + +func TestChromeCollector(t *testing.T) { + srv := fakeCDP(t) + defer srv.Close() + + c := NewChromeCollector(staticUpstream("ws" + strings.TrimPrefix(srv.URL, "http"))) + w := &Writer{} + require.NoError(t, c.Collect(context.Background(), w)) + out := string(w.Bytes()) + + assert.Contains(t, out, "kernel_chromium_up 1\n") + assert.Contains(t, out, `kernel_chromium_info{product="Chrome/145.0.7632.75"} 1`) + assert.Contains(t, out, "kernel_chromium_pages 2\n") + + // Cumulative le buckets from Chrome's disjoint buckets, overflow folded + // into +Inf. + fcp := `kernel_chromium_uma_bucket{histogram="PageLoad.PaintTiming.NavigationToFirstContentfulPaint",le=` + assert.Contains(t, out, fcp+`"638"} 2`) + assert.Contains(t, out, fcp+`"862"} 3`) + assert.Contains(t, out, fcp+`"+Inf"} 4`) + assert.Contains(t, out, `kernel_chromium_uma_sum{histogram="PageLoad.PaintTiming.NavigationToFirstContentfulPaint"} 21921`) + assert.Contains(t, out, `kernel_chromium_uma_count{histogram="PageLoad.PaintTiming.NavigationToFirstContentfulPaint"} 4`) + assert.NotContains(t, out, "UserInitiated") +} + +func TestChromeCollectorDown(t *testing.T) { + c := NewChromeCollector(staticUpstream("")) + w := &Writer{} + require.Error(t, c.Collect(context.Background(), w)) + assert.Contains(t, string(w.Bytes()), "kernel_chromium_up 0\n") +} diff --git a/server/lib/metrics/gpu.go b/server/lib/metrics/gpu.go new file mode 100644 index 00000000..c850da0e --- /dev/null +++ b/server/lib/metrics/gpu.go @@ -0,0 +1,197 @@ +package metrics + +import ( + "context" + "encoding/xml" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +// GPUCollector reports GPU presence and, when a GPU is attached, vGPU +// license state and utilization read from nvidia-smi. On VMs without the +// NVIDIA driver it emits only kernel_gpu_present 0. +type GPUCollector struct { + devicesDir string + querySMI func(ctx context.Context) ([]byte, error) + now func() time.Time +} + +func NewGPUCollector() *GPUCollector { + return &GPUCollector{ + devicesDir: "/proc/driver/nvidia/gpus", + querySMI: func(ctx context.Context) ([]byte, error) { + return exec.CommandContext(ctx, "nvidia-smi", "-q", "-x").Output() + }, + now: time.Now, + } +} + +func (c *GPUCollector) Name() string { return "gpu" } + +func (c *GPUCollector) Collect(ctx context.Context, w *Writer) error { + w.Metric("kernel_gpu_present", "Whether an NVIDIA GPU is attached to this VM.", "gauge") + entries, err := os.ReadDir(c.devicesDir) + if err != nil || len(entries) == 0 { + w.Sample("kernel_gpu_present", nil, 0) + return nil + } + w.Sample("kernel_gpu_present", nil, 1) + + out, err := c.querySMI(ctx) + if err != nil { + return fmt.Errorf("nvidia-smi: %w", err) + } + stats, err := parseNvidiaSMI(out, c.now()) + if err != nil { + return err + } + + w.Metric("kernel_gpu_info", "GPU device and vGPU license info; value is always 1.", "gauge") + w.Sample("kernel_gpu_info", []Label{ + {"product", stats.Product}, + {"licensed_product", stats.LicensedProduct}, + {"driver_version", stats.DriverVersion}, + }, 1) + + w.Metric("kernel_gpu_licensed", "Whether the vGPU software license is active.", "gauge") + w.Sample("kernel_gpu_licensed", nil, boolToFloat(stats.Licensed)) + + if stats.LicenseExpiry != nil { + w.Metric("kernel_gpu_license_expiry_seconds", + "Seconds until the vGPU license lease expires. The license daemon renews the lease continuously, so a value trending to zero means renewal is failing.", + "gauge") + w.Sample("kernel_gpu_license_expiry_seconds", nil, stats.LicenseExpiry.Sub(c.now()).Seconds()) + } + + utilization := []struct { + name string + val *float64 + }{ + {"kernel_gpu_utilization_percent", stats.GPUUtil}, + {"kernel_gpu_memory_utilization_percent", stats.MemoryUtil}, + {"kernel_gpu_encoder_utilization_percent", stats.EncoderUtil}, + {"kernel_gpu_decoder_utilization_percent", stats.DecoderUtil}, + } + for _, u := range utilization { + if u.val != nil { + w.Metric(u.name, "GPU utilization reported by nvidia-smi.", "gauge") + w.Sample(u.name, nil, *u.val) + } + } + if stats.MemoryUsedBytes != nil { + w.Metric("kernel_gpu_memory_used_bytes", "GPU framebuffer memory in use.", "gauge") + w.Sample("kernel_gpu_memory_used_bytes", nil, *stats.MemoryUsedBytes) + } + if stats.MemoryTotalBytes != nil { + w.Metric("kernel_gpu_memory_total_bytes", "Total GPU framebuffer memory.", "gauge") + w.Sample("kernel_gpu_memory_total_bytes", nil, *stats.MemoryTotalBytes) + } + return nil +} + +type gpuStats struct { + Product string + DriverVersion string + LicensedProduct string + Licensed bool + LicenseExpiry *time.Time + GPUUtil *float64 + MemoryUtil *float64 + EncoderUtil *float64 + DecoderUtil *float64 + MemoryUsedBytes *float64 + MemoryTotalBytes *float64 +} + +type smiLog struct { + DriverVersion string `xml:"driver_version"` + GPUs []struct { + ProductName string `xml:"product_name"` + Licensed struct { + ProductName string `xml:"licensed_product_name"` + Status string `xml:"license_status"` + } `xml:"vgpu_software_licensed_product"` + FBMemory struct { + Total string `xml:"total"` + Used string `xml:"used"` + } `xml:"fb_memory_usage"` + Utilization struct { + GPU string `xml:"gpu_util"` + Memory string `xml:"memory_util"` + Encoder string `xml:"encoder_util"` + Decoder string `xml:"decoder_util"` + } `xml:"utilization"` + } `xml:"gpu"` +} + +// licenseExpiryRe matches the expiry timestamp inside a license status like +// "Licensed (Expiry: 2026-7-7 15:58:42 GMT)". nvidia-smi does not zero-pad +// date or time components. +var licenseExpiryRe = regexp.MustCompile(`Expiry: (\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2} \w+)`) + +func parseNvidiaSMI(out []byte, now time.Time) (*gpuStats, error) { + var log smiLog + if err := xml.Unmarshal(out, &log); err != nil { + return nil, fmt.Errorf("parse nvidia-smi xml: %w", err) + } + if len(log.GPUs) == 0 { + return nil, fmt.Errorf("nvidia-smi reported no GPUs") + } + gpu := log.GPUs[0] + + stats := &gpuStats{ + Product: gpu.ProductName, + DriverVersion: log.DriverVersion, + LicensedProduct: gpu.Licensed.ProductName, + Licensed: strings.HasPrefix(gpu.Licensed.Status, "Licensed"), + } + if m := licenseExpiryRe.FindStringSubmatch(gpu.Licensed.Status); m != nil { + if t, err := time.Parse("2006-1-2 15:4:5 MST", m[1]); err == nil { + stats.LicenseExpiry = &t + } + } + stats.GPUUtil = parsePercent(gpu.Utilization.GPU) + stats.MemoryUtil = parsePercent(gpu.Utilization.Memory) + stats.EncoderUtil = parsePercent(gpu.Utilization.Encoder) + stats.DecoderUtil = parsePercent(gpu.Utilization.Decoder) + stats.MemoryUsedBytes = parseMiB(gpu.FBMemory.Used) + stats.MemoryTotalBytes = parseMiB(gpu.FBMemory.Total) + return stats, nil +} + +// parsePercent parses nvidia-smi values like "42 %". Returns nil for +// missing or "N/A" values. +func parsePercent(s string) *float64 { + v, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "%")), 64) + if err != nil { + return nil + } + return &v +} + +// parseMiB parses nvidia-smi memory values like "406 MiB" into bytes. +// Returns nil for missing or "N/A" values. +func parseMiB(s string) *float64 { + fields := strings.Fields(s) + if len(fields) != 2 || fields[1] != "MiB" { + return nil + } + v, err := strconv.ParseFloat(fields[0], 64) + if err != nil { + return nil + } + v *= 1024 * 1024 + return &v +} + +func boolToFloat(b bool) float64 { + if b { + return 1 + } + return 0 +} diff --git a/server/lib/metrics/gpu_test.go b/server/lib/metrics/gpu_test.go new file mode 100644 index 00000000..05159f65 --- /dev/null +++ b/server/lib/metrics/gpu_test.go @@ -0,0 +1,104 @@ +package metrics + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Trimmed from real `nvidia-smi -q -x` output on a vGPU browser VM. +const smiLicensed = ` + + 580.105.08 + 1 + + NVIDIA L40S-2Q + + NVIDIA RTX Virtual Workstation + Licensed (Expiry: 2026-7-7 16:1:44 GMT) + + + 2048 MiB + 458 MiB + 406 MiB + 1183 MiB + + + 7 % + 2 % + 0 % + N/A + + +` + +func TestParseNvidiaSMILicensed(t *testing.T) { + now := time.Date(2026, 7, 7, 15, 46, 44, 0, time.UTC) + stats, err := parseNvidiaSMI([]byte(smiLicensed), now) + require.NoError(t, err) + + assert.Equal(t, "NVIDIA L40S-2Q", stats.Product) + assert.Equal(t, "580.105.08", stats.DriverVersion) + assert.Equal(t, "NVIDIA RTX Virtual Workstation", stats.LicensedProduct) + assert.True(t, stats.Licensed) + require.NotNil(t, stats.LicenseExpiry) + assert.Equal(t, 15*time.Minute, stats.LicenseExpiry.Sub(now)) + + require.NotNil(t, stats.GPUUtil) + assert.Equal(t, 7.0, *stats.GPUUtil) + require.NotNil(t, stats.EncoderUtil) + assert.Equal(t, 0.0, *stats.EncoderUtil) + assert.Nil(t, stats.DecoderUtil) + + require.NotNil(t, stats.MemoryUsedBytes) + assert.Equal(t, 406.0*1024*1024, *stats.MemoryUsedBytes) + require.NotNil(t, stats.MemoryTotalBytes) + assert.Equal(t, 2048.0*1024*1024, *stats.MemoryTotalBytes) +} + +func TestParseNvidiaSMIUnlicensed(t *testing.T) { + xml := strings.Replace(smiLicensed, + "Licensed (Expiry: 2026-7-7 16:1:44 GMT)", "Unlicensed (Restricted)", 1) + stats, err := parseNvidiaSMI([]byte(xml), time.Now()) + require.NoError(t, err) + assert.False(t, stats.Licensed) + assert.Nil(t, stats.LicenseExpiry) +} + +func TestGPUCollectorNoGPU(t *testing.T) { + c := NewGPUCollector() + c.devicesDir = t.TempDir() // exists but empty + + w := &Writer{} + require.NoError(t, c.Collect(context.Background(), w)) + assert.Contains(t, string(w.Bytes()), "kernel_gpu_present 0\n") + assert.NotContains(t, string(w.Bytes()), "kernel_gpu_licensed") +} + +func TestGPUCollectorLicensed(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir, "0000:00:07.0"), 0o755)) + + c := NewGPUCollector() + c.devicesDir = dir + c.querySMI = func(context.Context) ([]byte, error) { return []byte(smiLicensed), nil } + c.now = func() time.Time { return time.Date(2026, 7, 7, 15, 56, 44, 0, time.UTC) } + + w := &Writer{} + require.NoError(t, c.Collect(context.Background(), w)) + out := string(w.Bytes()) + + assert.Contains(t, out, "kernel_gpu_present 1\n") + assert.Contains(t, out, `kernel_gpu_info{product="NVIDIA L40S-2Q",licensed_product="NVIDIA RTX Virtual Workstation",driver_version="580.105.08"} 1`) + assert.Contains(t, out, "kernel_gpu_licensed 1\n") + assert.Contains(t, out, "kernel_gpu_license_expiry_seconds 300\n") + assert.Contains(t, out, "kernel_gpu_utilization_percent 7\n") + assert.Contains(t, out, "kernel_gpu_memory_used_bytes 4.25721856e+08\n") + assert.NotContains(t, out, "kernel_gpu_decoder_utilization_percent") +} diff --git a/server/lib/metrics/metrics.go b/server/lib/metrics/metrics.go new file mode 100644 index 00000000..3e58e406 --- /dev/null +++ b/server/lib/metrics/metrics.go @@ -0,0 +1,98 @@ +// Package metrics exposes VM- and browser-level metrics in the Prometheus +// text exposition format, intended to be scraped by an external collector +// and aggregated across VMs. Metrics carry no per-session or per-user +// labels; instance identity is expected to be attached (and later +// aggregated away) by the scraper. +package metrics + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "math" + "net/http" + "strconv" + "sync" + "time" +) + +// Collector renders one group of metrics for a single scrape. +type Collector interface { + // Name identifies the collector in error logs. + Name() string + // Collect appends samples to w. Returning an error does not fail the + // scrape; the other collectors' output is still served. + Collect(ctx context.Context, w *Writer) error +} + +// Label is a single Prometheus label pair. +type Label struct { + Name string + Value string +} + +// Writer accumulates Prometheus text-format output. +type Writer struct { + buf bytes.Buffer +} + +// Metric writes the HELP and TYPE header lines for a metric family. +func (w *Writer) Metric(name, help, typ string) { + fmt.Fprintf(&w.buf, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ) +} + +// Sample writes one sample line. name may carry a family suffix such as +// _bucket, _sum, or _count for histogram families. +func (w *Writer) Sample(name string, labels []Label, value float64) { + w.buf.WriteString(name) + if len(labels) > 0 { + w.buf.WriteByte('{') + for i, l := range labels { + if i > 0 { + w.buf.WriteByte(',') + } + // %q emits the exposition-format escapes: \\, \", and \n. + fmt.Fprintf(&w.buf, "%s=%q", l.Name, l.Value) + } + w.buf.WriteByte('}') + } + w.buf.WriteByte(' ') + w.buf.WriteString(formatValue(value)) + w.buf.WriteByte('\n') +} + +func (w *Writer) Bytes() []byte { + return w.buf.Bytes() +} + +func formatValue(v float64) string { + if math.IsInf(v, 1) { + return "+Inf" + } + return strconv.FormatFloat(v, 'g', -1, 64) +} + +const scrapeTimeout = 10 * time.Second + +// Handler serves GET /metrics. Scrapes are serialized: concurrent requests +// wait rather than probing Chrome and nvidia-smi in parallel. +func Handler(log *slog.Logger, collectors ...Collector) http.Handler { + var mu sync.Mutex + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + + ctx, cancel := context.WithTimeout(r.Context(), scrapeTimeout) + defer cancel() + + w := &Writer{} + for _, c := range collectors { + if err := c.Collect(ctx, w); err != nil { + log.Error("metrics collector failed", "collector", c.Name(), "err", err) + } + } + rw.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + rw.Write(w.Bytes()) + }) +} diff --git a/server/lib/metrics/metrics_test.go b/server/lib/metrics/metrics_test.go new file mode 100644 index 00000000..e8594042 --- /dev/null +++ b/server/lib/metrics/metrics_test.go @@ -0,0 +1,60 @@ +package metrics + +import ( + "context" + "errors" + "log/slog" + "math" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWriterSample(t *testing.T) { + w := &Writer{} + w.Metric("test_metric", "A test metric.", "gauge") + w.Sample("test_metric", nil, 42) + w.Sample("test_metric", []Label{{"mode", "user"}, {"mount", "/"}}, 1.5) + w.Sample("test_metric", []Label{{"name", "quote\"back\\slash\nnewline"}}, math.Inf(1)) + + assert.Equal(t, `# HELP test_metric A test metric. +# TYPE test_metric gauge +test_metric 42 +test_metric{mode="user",mount="/"} 1.5 +test_metric{name="quote\"back\\slash\nnewline"} +Inf +`, string(w.Bytes())) +} + +type stubCollector struct { + name string + fn func(w *Writer) error +} + +func (s *stubCollector) Name() string { return s.name } +func (s *stubCollector) Collect(_ context.Context, w *Writer) error { + return s.fn(w) +} + +func TestHandlerServesAllCollectorsDespiteError(t *testing.T) { + failing := &stubCollector{name: "bad", fn: func(w *Writer) error { + w.Metric("bad_up", "Whether bad works.", "gauge") + w.Sample("bad_up", nil, 0) + return errors.New("boom") + }} + ok := &stubCollector{name: "good", fn: func(w *Writer) error { + w.Metric("good_total", "Good things.", "counter") + w.Sample("good_total", nil, 3) + return nil + }} + + h := Handler(slog.New(slog.DiscardHandler), failing, ok) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest("GET", "/metrics", nil)) + + require.Equal(t, 200, rec.Code) + assert.Equal(t, "text/plain; version=0.0.4; charset=utf-8", rec.Header().Get("Content-Type")) + assert.Contains(t, rec.Body.String(), "bad_up 0\n") + assert.Contains(t, rec.Body.String(), "good_total 3\n") +} diff --git a/server/lib/metrics/system.go b/server/lib/metrics/system.go new file mode 100644 index 00000000..859b872b --- /dev/null +++ b/server/lib/metrics/system.go @@ -0,0 +1,205 @@ +package metrics + +import ( + "context" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" +) + +// SystemCollector reports VM-level resource metrics from /proc plus +// aggregate stats for the Chromium process tree. +type SystemCollector struct { + procDir string + rootPath string +} + +func NewSystemCollector() *SystemCollector { + return &SystemCollector{procDir: "/proc", rootPath: "/"} +} + +func (c *SystemCollector) Name() string { return "system" } + +// userHZ is the kernel USER_HZ /proc/stat tick rate. Fixed at 100 on Linux +// for all architectures this image builds for. +const userHZ = 100 + +func (c *SystemCollector) Collect(ctx context.Context, w *Writer) error { + if data, err := os.ReadFile(filepath.Join(c.procDir, "stat")); err == nil { + if modes := parseProcStatCPU(string(data)); modes != nil { + w.Metric("kernel_vm_cpu_seconds_total", "Cumulative CPU time by mode across all cores.", "counter") + for _, m := range modes { + w.Sample("kernel_vm_cpu_seconds_total", []Label{{"mode", m.mode}}, m.seconds) + } + } + } + + if data, err := os.ReadFile(filepath.Join(c.procDir, "meminfo")); err == nil { + mem := parseMeminfo(string(data)) + if v, ok := mem["MemTotal"]; ok { + w.Metric("kernel_vm_memory_total_bytes", "Total VM memory.", "gauge") + w.Sample("kernel_vm_memory_total_bytes", nil, v) + } + if v, ok := mem["MemAvailable"]; ok { + w.Metric("kernel_vm_memory_available_bytes", "Estimated memory available for new workloads.", "gauge") + w.Sample("kernel_vm_memory_available_bytes", nil, v) + } + } + + if data, err := os.ReadFile(filepath.Join(c.procDir, "loadavg")); err == nil { + if fields := strings.Fields(string(data)); len(fields) > 0 { + if v, err := strconv.ParseFloat(fields[0], 64); err == nil { + w.Metric("kernel_vm_load1", "1-minute load average.", "gauge") + w.Sample("kernel_vm_load1", nil, v) + } + } + } + + if data, err := os.ReadFile(filepath.Join(c.procDir, "uptime")); err == nil { + if fields := strings.Fields(string(data)); len(fields) > 0 { + if v, err := strconv.ParseFloat(fields[0], 64); err == nil { + // CLOCK_MONOTONIC pauses while a scale-to-zero VM is + // suspended, so this measures active runtime, not wall age. + w.Metric("kernel_vm_uptime_seconds", "VM active runtime; excludes time spent suspended.", "gauge") + w.Sample("kernel_vm_uptime_seconds", nil, v) + } + } + } + + if data, err := os.ReadFile(filepath.Join(c.procDir, "net", "dev")); err == nil { + rx, tx := parseNetDev(string(data)) + w.Metric("kernel_vm_network_receive_bytes_total", "Bytes received on non-loopback interfaces.", "counter") + w.Sample("kernel_vm_network_receive_bytes_total", nil, rx) + w.Metric("kernel_vm_network_transmit_bytes_total", "Bytes transmitted on non-loopback interfaces.", "counter") + w.Sample("kernel_vm_network_transmit_bytes_total", nil, tx) + } + + var fs syscall.Statfs_t + if err := syscall.Statfs(c.rootPath, &fs); err == nil { + bsize := float64(fs.Bsize) + w.Metric("kernel_vm_disk_total_bytes", "Filesystem size of the root mount.", "gauge") + w.Sample("kernel_vm_disk_total_bytes", []Label{{"mount", c.rootPath}}, float64(fs.Blocks)*bsize) + w.Metric("kernel_vm_disk_free_bytes", "Filesystem space available to unprivileged users on the root mount.", "gauge") + w.Sample("kernel_vm_disk_free_bytes", []Label{{"mount", c.rootPath}}, float64(fs.Bavail)*bsize) + } + + procs, rssBytes := c.chromiumProcesses() + w.Metric("kernel_chromium_processes", "Number of running Chromium processes (browser, renderers, utilities).", "gauge") + w.Sample("kernel_chromium_processes", nil, float64(procs)) + w.Metric("kernel_chromium_memory_rss_bytes", "Total resident memory of all Chromium processes.", "gauge") + w.Sample("kernel_chromium_memory_rss_bytes", nil, rssBytes) + + return nil +} + +type cpuMode struct { + mode string + seconds float64 +} + +// parseProcStatCPU reads the aggregate "cpu" line of /proc/stat. Fields: +// user nice system idle iowait irq softirq steal. nice folds into user; +// irq and softirq fold into system. +func parseProcStatCPU(data string) []cpuMode { + for _, line := range strings.Split(data, "\n") { + if !strings.HasPrefix(line, "cpu ") { + continue + } + fields := strings.Fields(line)[1:] + if len(fields) < 8 { + return nil + } + ticks := make([]float64, 8) + for i := range ticks { + v, err := strconv.ParseFloat(fields[i], 64) + if err != nil { + return nil + } + ticks[i] = v + } + return []cpuMode{ + {"user", (ticks[0] + ticks[1]) / userHZ}, + {"system", (ticks[2] + ticks[5] + ticks[6]) / userHZ}, + {"idle", ticks[3] / userHZ}, + {"iowait", ticks[4] / userHZ}, + {"steal", ticks[7] / userHZ}, + } + } + return nil +} + +// parseMeminfo returns /proc/meminfo values in bytes. +func parseMeminfo(data string) map[string]float64 { + out := make(map[string]float64) + for _, line := range strings.Split(data, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + v, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + continue + } + out[strings.TrimSuffix(fields[0], ":")] = v * 1024 + } + return out +} + +// parseNetDev sums received and transmitted bytes across all non-loopback +// interfaces in /proc/net/dev. +func parseNetDev(data string) (rx, tx float64) { + for _, line := range strings.Split(data, "\n") { + name, stats, ok := strings.Cut(line, ":") + if !ok || strings.TrimSpace(name) == "lo" { + continue + } + fields := strings.Fields(stats) + if len(fields) < 9 { + continue + } + if v, err := strconv.ParseFloat(fields[0], 64); err == nil { + rx += v + } + if v, err := strconv.ParseFloat(fields[8], 64); err == nil { + tx += v + } + } + return rx, tx +} + +// chromiumProcesses counts processes whose comm is "chromium" and sums +// their resident memory. +func (c *SystemCollector) chromiumProcesses() (count int, rssBytes float64) { + entries, err := os.ReadDir(c.procDir) + if err != nil { + return 0, 0 + } + pageSize := float64(os.Getpagesize()) + for _, e := range entries { + if !e.IsDir() { + continue + } + if _, err := strconv.Atoi(e.Name()); err != nil { + continue + } + comm, err := os.ReadFile(filepath.Join(c.procDir, e.Name(), "comm")) + if err != nil || strings.TrimSpace(string(comm)) != "chromium" { + continue + } + count++ + statm, err := os.ReadFile(filepath.Join(c.procDir, e.Name(), "statm")) + if err != nil { + continue + } + fields := strings.Fields(string(statm)) + if len(fields) < 2 { + continue + } + if pages, err := strconv.ParseFloat(fields[1], 64); err == nil { + rssBytes += pages * pageSize + } + } + return count, rssBytes +} diff --git a/server/lib/metrics/system_test.go b/server/lib/metrics/system_test.go new file mode 100644 index 00000000..15eb9b9f --- /dev/null +++ b/server/lib/metrics/system_test.go @@ -0,0 +1,66 @@ +package metrics + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseProcStatCPU(t *testing.T) { + data := `cpu 1000 200 300 40000 500 60 70 80 0 0 +cpu0 500 100 150 20000 250 30 35 40 0 0 +` + modes := parseProcStatCPU(data) + require.Len(t, modes, 5) + byMode := map[string]float64{} + for _, m := range modes { + byMode[m.mode] = m.seconds + } + assert.Equal(t, 12.0, byMode["user"]) // (1000+200)/100 + assert.Equal(t, 4.3, byMode["system"]) // (300+60+70)/100 + assert.Equal(t, 400.0, byMode["idle"]) // 40000/100 + assert.Equal(t, 5.0, byMode["iowait"]) // 500/100 + assert.Equal(t, 0.8, byMode["steal"]) // 80/100 +} + +func TestParseMeminfo(t *testing.T) { + mem := parseMeminfo("MemTotal: 4014828 kB\nMemAvailable: 2534512 kB\nHugePages_Total: 0\n") + assert.Equal(t, 4014828.0*1024, mem["MemTotal"]) + assert.Equal(t, 2534512.0*1024, mem["MemAvailable"]) +} + +func TestParseNetDev(t *testing.T) { + data := `Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + lo: 999999 1000 0 0 0 0 0 0 999999 1000 0 0 0 0 0 0 + eth0: 1500000 12000 0 0 0 0 0 0 300000 8000 0 0 0 0 0 0 + eth1: 500000 2000 0 0 0 0 0 0 200000 1000 0 0 0 0 0 0 +` + rx, tx := parseNetDev(data) + assert.Equal(t, 2000000.0, rx) + assert.Equal(t, 500000.0, tx) +} + +func TestChromiumProcesses(t *testing.T) { + proc := t.TempDir() + mkProc := func(pid, comm, statm string) { + dir := filepath.Join(proc, pid) + require.NoError(t, os.Mkdir(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "comm"), []byte(comm+"\n"), 0o644)) + if statm != "" { + require.NoError(t, os.WriteFile(filepath.Join(dir, "statm"), []byte(statm), 0o644)) + } + } + mkProc("100", "chromium", "5000 1000 300 10 0 500 0") + mkProc("101", "chromium", "6000 2000 300 10 0 500 0") + mkProc("102", "chrome_crashpad", "100 50 30 1 0 5 0") + mkProc("103", "bash", "100 50 30 1 0 5 0") + + c := &SystemCollector{procDir: proc, rootPath: "/"} + count, rss := c.chromiumProcesses() + assert.Equal(t, 2, count) + assert.Equal(t, 3000.0*float64(os.Getpagesize()), rss) +} From c8c97436c53c07afe284827e25810da883d0ed39 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 7 Jul 2026 18:38:32 +0000 Subject: [PATCH 2/3] Expand curated UMA set with interaction, startup, and scroll metrics Add FID, interactions-per-page, browser-start-to-first-paint, and peak GPU memory during scroll. All four are recorded in the browser process, verified against a live browser after real usage; renderer/GPU-process histograms (EventLatency, Graphics.Smoothness, Blink.*) are not visible to Browser.getHistograms without a histogram sync, so they stay out. Co-Authored-By: Claude Opus 4.7 --- server/lib/metrics/chrome.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/server/lib/metrics/chrome.go b/server/lib/metrics/chrome.go index e1ef1cc8..9896697a 100644 --- a/server/lib/metrics/chrome.go +++ b/server/lib/metrics/chrome.go @@ -10,21 +10,36 @@ import ( ) // DefaultUMAHistograms is the curated set of Chrome UMA histograms exposed -// on the metrics endpoint. All are recorded per page load with millisecond -// units unless noted. Histograms with no samples yet are simply absent from -// the output. +// on the metrics endpoint. Units are milliseconds unless noted. Histograms +// with no samples yet are simply absent from the output. +// +// Only histograms recorded in the browser process are eligible: +// Browser.getHistograms cannot see renderer/GPU-process histograms (e.g. +// EventLatency.*, Graphics.Smoothness.*, Blink.*) because those only merge +// into the browser's recorder when a UMA log is staged or chrome://histograms +// is opened, neither of which happens on our images. var DefaultUMAHistograms = []string{ "PageLoad.PaintTiming.NavigationToFirstContentfulPaint", "PageLoad.PaintTiming.NavigationToLargestContentfulPaint2", "PageLoad.DocumentTiming.NavigationToDOMContentLoadedEventFired", "PageLoad.DocumentTiming.NavigationToLoadEventFired", "PageLoad.ParseTiming.NavigationToParseStart", - // INP: worst-case interaction latency, high percentile per page load. + // INP: worst-case interaction latency (input to next paint), high + // percentile per page load. "PageLoad.InteractiveTiming.UserInteractionLatency.HighPercentile2.MaxEventDuration", + // FID: input delay of the first interaction per page load. + "PageLoad.InteractiveTiming.FirstInputDelay4", + // Interactions per page load; unitless count. + "PageLoad.InteractiveTiming.NumInteractions", // CLS: unitless layout-shift score, not milliseconds. "PageLoad.LayoutInstability.MaxCumulativeShiftScore.SessionWindow.Gap1000ms.Max5000ms2", // CPU milliseconds consumed by the page across its lifetime. "PageLoad.Cpu.TotalUsage", + // Browser start to first web contents paint; one sample per browser + // start, measures how quickly a fresh VM renders content. + "Startup.FirstWebContents.NonEmptyPaint3", + // Peak GPU memory during scroll sequences, in MB. + "Memory.GPU.PeakMemoryUsage2.Scroll", } // DevToolsUpstream reports the current browser-level DevTools WebSocket URL. From f9931907f04028c4863cf053946122ac79d4e0be Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 10 Jul 2026 00:02:18 +0000 Subject: [PATCH 3/3] Re-bucket UMA histograms onto fixed bounds and harden the metrics handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - UMA histograms are re-bucketed onto small fixed per-histogram boundary grids (~10 bounds) at exposition time. Chrome's native buckets are sparse and exponential, so every VM previously emitted a different le set, which breaks cross-VM histogram aggregation and inflates cardinality. All fixed bounds are emitted, including empty ones. - The handler now gathers each collector's output into its own buffer and appends it only on success, so a failure can never leave a partially written metric family in the response. Each collector also gets its own deadline under the scrape budget so a slow one cannot starve the others. An unreachable browser is no longer a collector error: chrome writes up 0 and returns nil so the signal survives the discard-on-error policy. - The curated histograms are fetched in ~5 CDP round trips instead of 14 by covering the PageLoad family with one substring query. - New PSI gauges from /proc/pressure (some avg10 per resource), skipped gracefully on kernels without CONFIG_PSI — current guest kernels lack it, so these stay dormant until images enable it. - Hardening and doc fixes: +Inf uses max(count, bucket sum); UMA HELP spells out the non-millisecond units; license expiry parsing requires the GMT zone instead of silently zero-offsetting unknown zones; RSS sum documented as a shared-page-inflated upper bound; uptime comment corrected to CLOCK_BOOTTIME; unused parseNvidiaSMI param removed. Co-Authored-By: Claude Opus 4.7 --- server/lib/metrics/chrome.go | 158 +++++++++++++++++++++-------- server/lib/metrics/chrome_test.go | 33 +++++- server/lib/metrics/gpu.go | 16 +-- server/lib/metrics/gpu_test.go | 18 +++- server/lib/metrics/metrics.go | 29 +++++- server/lib/metrics/metrics_test.go | 8 +- server/lib/metrics/system.go | 45 +++++++- server/lib/metrics/system_test.go | 28 +++++ 8 files changed, 267 insertions(+), 68 deletions(-) diff --git a/server/lib/metrics/chrome.go b/server/lib/metrics/chrome.go index 9896697a..6b5c6f79 100644 --- a/server/lib/metrics/chrome.go +++ b/server/lib/metrics/chrome.go @@ -5,10 +5,33 @@ import ( "errors" "math" "strconv" + "strings" "github.com/kernel/kernel-images/server/lib/cdpclient" ) +// UMAHistogram is one curated Chrome histogram plus the fixed bucket +// boundaries it is re-bucketed onto for exposition. +type UMAHistogram struct { + Name string + Bounds []int64 +} + +// Fixed bucket boundary grids. Chrome's native buckets are exponential +// with per-histogram layouts and only non-empty buckets are reported, so +// exposing them directly would give every VM a different le set — which +// breaks cross-VM histogram aggregation and inflates cardinality. +// Re-bucketing onto small fixed grids makes every VM emit identical le +// sets at ~10 bounds per histogram. +var ( + boundsPageLoadMs = []int64{100, 250, 500, 1000, 2000, 4000, 8000, 15000, 30000, 60000} + boundsInteractionMs = []int64{10, 25, 50, 100, 200, 300, 500, 1000, 2500, 10000} + boundsCPUMs = []int64{50, 100, 250, 500, 1000, 2500, 5000, 15000, 60000} + boundsShiftScore = []int64{1, 5, 10, 25, 50, 100, 250, 1000} + boundsCount = []int64{1, 2, 5, 10, 25, 50, 100, 250} + boundsMB = []int64{16, 32, 64, 128, 256, 512, 1024, 2048} +) + // DefaultUMAHistograms is the curated set of Chrome UMA histograms exposed // on the metrics endpoint. Units are milliseconds unless noted. Histograms // with no samples yet are simply absent from the output. @@ -18,28 +41,28 @@ import ( // EventLatency.*, Graphics.Smoothness.*, Blink.*) because those only merge // into the browser's recorder when a UMA log is staged or chrome://histograms // is opened, neither of which happens on our images. -var DefaultUMAHistograms = []string{ - "PageLoad.PaintTiming.NavigationToFirstContentfulPaint", - "PageLoad.PaintTiming.NavigationToLargestContentfulPaint2", - "PageLoad.DocumentTiming.NavigationToDOMContentLoadedEventFired", - "PageLoad.DocumentTiming.NavigationToLoadEventFired", - "PageLoad.ParseTiming.NavigationToParseStart", +var DefaultUMAHistograms = []UMAHistogram{ + {Name: "PageLoad.PaintTiming.NavigationToFirstContentfulPaint", Bounds: boundsPageLoadMs}, + {Name: "PageLoad.PaintTiming.NavigationToLargestContentfulPaint2", Bounds: boundsPageLoadMs}, + {Name: "PageLoad.DocumentTiming.NavigationToDOMContentLoadedEventFired", Bounds: boundsPageLoadMs}, + {Name: "PageLoad.DocumentTiming.NavigationToLoadEventFired", Bounds: boundsPageLoadMs}, + {Name: "PageLoad.ParseTiming.NavigationToParseStart", Bounds: boundsPageLoadMs}, // INP: worst-case interaction latency (input to next paint), high // percentile per page load. - "PageLoad.InteractiveTiming.UserInteractionLatency.HighPercentile2.MaxEventDuration", + {Name: "PageLoad.InteractiveTiming.UserInteractionLatency.HighPercentile2.MaxEventDuration", Bounds: boundsInteractionMs}, // FID: input delay of the first interaction per page load. - "PageLoad.InteractiveTiming.FirstInputDelay4", + {Name: "PageLoad.InteractiveTiming.FirstInputDelay4", Bounds: boundsInteractionMs}, // Interactions per page load; unitless count. - "PageLoad.InteractiveTiming.NumInteractions", + {Name: "PageLoad.InteractiveTiming.NumInteractions", Bounds: boundsCount}, // CLS: unitless layout-shift score, not milliseconds. - "PageLoad.LayoutInstability.MaxCumulativeShiftScore.SessionWindow.Gap1000ms.Max5000ms2", + {Name: "PageLoad.LayoutInstability.MaxCumulativeShiftScore.SessionWindow.Gap1000ms.Max5000ms2", Bounds: boundsShiftScore}, // CPU milliseconds consumed by the page across its lifetime. - "PageLoad.Cpu.TotalUsage", + {Name: "PageLoad.Cpu.TotalUsage", Bounds: boundsCPUMs}, // Browser start to first web contents paint; one sample per browser // start, measures how quickly a fresh VM renders content. - "Startup.FirstWebContents.NonEmptyPaint3", + {Name: "Startup.FirstWebContents.NonEmptyPaint3", Bounds: boundsPageLoadMs}, // Peak GPU memory during scroll sequences, in MB. - "Memory.GPU.PeakMemoryUsage2.Scroll", + {Name: "Memory.GPU.PeakMemoryUsage2.Scroll", Bounds: boundsMB}, } // DevToolsUpstream reports the current browser-level DevTools WebSocket URL. @@ -54,7 +77,7 @@ type DevToolsUpstream interface { // sessions, so scrapes are invisible to automation running in the browser. type ChromeCollector struct { upstream DevToolsUpstream - histograms []string + histograms []UMAHistogram } func NewChromeCollector(upstream DevToolsUpstream) *ChromeCollector { @@ -68,8 +91,12 @@ func (c *ChromeCollector) Collect(ctx context.Context, w *Writer) error { client, version, err := c.dial(ctx) if err != nil { + // An unreachable browser is a routine, load-bearing signal (the + // browser may simply be restarting), not a collector failure: + // return nil so the up sample survives the handler's + // discard-on-error policy. w.Sample("kernel_chromium_up", nil, 0) - return err + return nil } defer client.Close() w.Sample("kernel_chromium_up", nil, 1) @@ -82,23 +109,59 @@ func (c *ChromeCollector) Collect(ctx context.Context, w *Writer) error { w.Sample("kernel_chromium_pages", nil, float64(pages)) } + fetched, err := c.fetchHistograms(ctx, client) + if err != nil { + return err + } w.Metric("kernel_chromium_uma", - "Chrome UMA histogram, cumulative since browser start. Units follow the UMA definition; PageLoad timings are milliseconds.", + "Chrome UMA histogram, cumulative since browser start, re-bucketed onto fixed bounds. Units follow the UMA definition: PageLoad timings are milliseconds, NumInteractions is a count, CLS is a unitless score, Memory.GPU.* is MB.", "histogram") - for _, name := range c.histograms { - matches, err := client.GetHistograms(ctx, name) + for _, uma := range c.histograms { + if h, ok := fetched[uma.Name]; ok { + writeUMAHistogram(w, h, uma.Bounds) + } + } + return nil +} + +// fetchHistograms retrieves the curated histograms in as few CDP round +// trips as possible: one substring query covers the whole PageLoad family, +// plus one query per remaining name. The query is a substring filter, so +// results also contain suffixed variants — only exact names are kept. +func (c *ChromeCollector) fetchHistograms(ctx context.Context, client *cdpclient.Client) (map[string]cdpclient.Histogram, error) { + const pageLoadPrefix = "PageLoad." + + queries := make([]string, 0, len(c.histograms)) + hasPageLoad := false + for _, uma := range c.histograms { + if strings.HasPrefix(uma.Name, pageLoadPrefix) { + hasPageLoad = true + continue + } + queries = append(queries, uma.Name) + } + if hasPageLoad { + queries = append(queries, pageLoadPrefix) + } + + wanted := make(map[string]struct{}, len(c.histograms)) + for _, uma := range c.histograms { + wanted[uma.Name] = struct{}{} + } + + fetched := make(map[string]cdpclient.Histogram, len(c.histograms)) + for _, query := range queries { + matches, err := client.GetHistograms(ctx, query) if err != nil { - return err + return nil, err } for _, h := range matches { - // The query is a substring filter, so it also returns - // suffixed variants of the requested histogram. - if h.Name == name { - writeUMAHistogram(w, h) + if _, ok := wanted[h.Name]; ok { + fetched[h.Name] = h } } } - return nil + return fetched, nil } func (c *ChromeCollector) dial(ctx context.Context) (*cdpclient.Client, *cdpclient.BrowserVersion, error) { @@ -118,27 +181,40 @@ func (c *ChromeCollector) dial(ctx context.Context) (*cdpclient.Client, *cdpclie return client, version, nil } -// writeUMAHistogram emits one UMA histogram as a Prometheus histogram with a -// "histogram" label carrying the UMA name. Chrome reports sparse, disjoint -// [low, high) buckets; the upper bound maps to a cumulative le bound. The -// overflow bucket (high = MaxInt32) folds into +Inf. -func writeUMAHistogram(w *Writer, h cdpclient.Histogram) { - labels := func(le string) []Label { - return []Label{{"histogram", h.Name}, {"le", le}} - } - count := h.Count - if count == 0 { - for _, b := range h.Buckets { - count += b.Count - } - } - cumulative := int64(0) +// writeUMAHistogram emits one UMA histogram as a Prometheus histogram with +// a "histogram" label carrying the UMA name. Chrome's sparse [low, high) +// buckets are re-bucketed onto the fixed bounds: each chrome bucket counts +// toward the smallest fixed bound >= its upper edge (rounding a straddling +// bucket up, biased at most one fixed bucket wide). All fixed bounds are +// emitted, including empty ones, so every VM produces an identical le set. +func writeUMAHistogram(w *Writer, h cdpclient.Histogram, bounds []int64) { + cumulative := make([]int64, len(bounds)) + bucketSum := int64(0) for _, b := range h.Buckets { + bucketSum += b.Count if b.High >= math.MaxInt32 { + // Overflow bucket: counted in +Inf only. continue } - cumulative += b.Count - w.Sample("kernel_chromium_uma_bucket", labels(strconv.FormatInt(b.High, 10)), float64(cumulative)) + for i, bound := range bounds { + if b.High <= bound { + cumulative[i] += b.Count + break + } + } + } + for i := 1; i < len(cumulative); i++ { + cumulative[i] += cumulative[i-1] + } + // Chrome's count and its bucket sum are read from live memory and can + // disagree; taking the max keeps +Inf >= the last bucket. + count := max(h.Count, bucketSum) + + labels := func(le string) []Label { + return []Label{{"histogram", h.Name}, {"le", le}} + } + for i, bound := range bounds { + w.Sample("kernel_chromium_uma_bucket", labels(strconv.FormatInt(bound, 10)), float64(cumulative[i])) } w.Sample("kernel_chromium_uma_bucket", labels("+Inf"), float64(count)) w.Sample("kernel_chromium_uma_sum", []Label{{"histogram", h.Name}}, float64(h.Sum)) diff --git a/server/lib/metrics/chrome_test.go b/server/lib/metrics/chrome_test.go index c33ae743..324908cd 100644 --- a/server/lib/metrics/chrome_test.go +++ b/server/lib/metrics/chrome_test.go @@ -51,6 +51,12 @@ func fakeCDP(t *testing.T) *httptest.Server { {"targetId": "C", "type": "service_worker"}, }} case "Browser.getHistograms": + if req.Params.Query == "Fail.Me" { + resp, err := json.Marshal(map[string]any{"id": req.ID, "error": map[string]any{"code": -32000, "message": "boom"}}) + require.NoError(t, err) + require.NoError(t, conn.Write(ctx, websocket.MessageText, resp)) + continue + } histograms := []map[string]any{} if strings.Contains("PageLoad.PaintTiming.NavigationToFirstContentfulPaint", req.Params.Query) { histograms = append(histograms, @@ -94,20 +100,37 @@ func TestChromeCollector(t *testing.T) { assert.Contains(t, out, `kernel_chromium_info{product="Chrome/145.0.7632.75"} 1`) assert.Contains(t, out, "kernel_chromium_pages 2\n") - // Cumulative le buckets from Chrome's disjoint buckets, overflow folded - // into +Inf. + // Chrome's buckets ([606,638):2, [819,862):1, overflow:1) re-bucket + // onto the fixed grid: both finite buckets land at le=1000, the + // overflow lands in +Inf, and every fixed bound is emitted so all VMs + // produce identical le sets. fcp := `kernel_chromium_uma_bucket{histogram="PageLoad.PaintTiming.NavigationToFirstContentfulPaint",le=` - assert.Contains(t, out, fcp+`"638"} 2`) - assert.Contains(t, out, fcp+`"862"} 3`) + assert.Contains(t, out, fcp+`"100"} 0`) + assert.Contains(t, out, fcp+`"500"} 0`) + assert.Contains(t, out, fcp+`"1000"} 3`) + assert.Contains(t, out, fcp+`"60000"} 3`) assert.Contains(t, out, fcp+`"+Inf"} 4`) + assert.NotContains(t, out, `le="638"`) assert.Contains(t, out, `kernel_chromium_uma_sum{histogram="PageLoad.PaintTiming.NavigationToFirstContentfulPaint"} 21921`) assert.Contains(t, out, `kernel_chromium_uma_count{histogram="PageLoad.PaintTiming.NavigationToFirstContentfulPaint"} 4`) assert.NotContains(t, out, "UserInitiated") } func TestChromeCollectorDown(t *testing.T) { + // An unreachable browser is not a collector error: the up sample must + // survive the handler's discard-on-error policy. c := NewChromeCollector(staticUpstream("")) w := &Writer{} - require.Error(t, c.Collect(context.Background(), w)) + require.NoError(t, c.Collect(context.Background(), w)) assert.Contains(t, string(w.Bytes()), "kernel_chromium_up 0\n") } + +func TestChromeCollectorHistogramFailure(t *testing.T) { + srv := fakeCDP(t) + defer srv.Close() + + c := NewChromeCollector(staticUpstream("ws" + strings.TrimPrefix(srv.URL, "http"))) + c.histograms = []UMAHistogram{{Name: "Fail.Me", Bounds: boundsPageLoadMs}} + w := &Writer{} + require.Error(t, c.Collect(context.Background(), w)) +} diff --git a/server/lib/metrics/gpu.go b/server/lib/metrics/gpu.go index c850da0e..07ecac84 100644 --- a/server/lib/metrics/gpu.go +++ b/server/lib/metrics/gpu.go @@ -46,7 +46,7 @@ func (c *GPUCollector) Collect(ctx context.Context, w *Writer) error { if err != nil { return fmt.Errorf("nvidia-smi: %w", err) } - stats, err := parseNvidiaSMI(out, c.now()) + stats, err := parseNvidiaSMI(out) if err != nil { return err } @@ -131,10 +131,13 @@ type smiLog struct { // licenseExpiryRe matches the expiry timestamp inside a license status like // "Licensed (Expiry: 2026-7-7 15:58:42 GMT)". nvidia-smi does not zero-pad -// date or time components. -var licenseExpiryRe = regexp.MustCompile(`Expiry: (\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2} \w+)`) +// date or time components. The zone is captured separately and required to +// be GMT: time.Parse would silently give an unknown abbreviation a zero +// offset, so a driver that starts emitting another zone must fail parsing +// loudly (no expiry metric) rather than skew it. +var licenseExpiryRe = regexp.MustCompile(`Expiry: (\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) (\w+)`) -func parseNvidiaSMI(out []byte, now time.Time) (*gpuStats, error) { +func parseNvidiaSMI(out []byte) (*gpuStats, error) { var log smiLog if err := xml.Unmarshal(out, &log); err != nil { return nil, fmt.Errorf("parse nvidia-smi xml: %w", err) @@ -142,6 +145,7 @@ func parseNvidiaSMI(out []byte, now time.Time) (*gpuStats, error) { if len(log.GPUs) == 0 { return nil, fmt.Errorf("nvidia-smi reported no GPUs") } + // Browser VMs get exactly one vGPU slice; only the first GPU is read. gpu := log.GPUs[0] stats := &gpuStats{ @@ -150,8 +154,8 @@ func parseNvidiaSMI(out []byte, now time.Time) (*gpuStats, error) { LicensedProduct: gpu.Licensed.ProductName, Licensed: strings.HasPrefix(gpu.Licensed.Status, "Licensed"), } - if m := licenseExpiryRe.FindStringSubmatch(gpu.Licensed.Status); m != nil { - if t, err := time.Parse("2006-1-2 15:4:5 MST", m[1]); err == nil { + if m := licenseExpiryRe.FindStringSubmatch(gpu.Licensed.Status); m != nil && m[2] == "GMT" { + if t, err := time.ParseInLocation("2006-1-2 15:4:5", m[1], time.UTC); err == nil { stats.LicenseExpiry = &t } } diff --git a/server/lib/metrics/gpu_test.go b/server/lib/metrics/gpu_test.go index 05159f65..2b45e69d 100644 --- a/server/lib/metrics/gpu_test.go +++ b/server/lib/metrics/gpu_test.go @@ -39,8 +39,7 @@ const smiLicensed = ` ` func TestParseNvidiaSMILicensed(t *testing.T) { - now := time.Date(2026, 7, 7, 15, 46, 44, 0, time.UTC) - stats, err := parseNvidiaSMI([]byte(smiLicensed), now) + stats, err := parseNvidiaSMI([]byte(smiLicensed)) require.NoError(t, err) assert.Equal(t, "NVIDIA L40S-2Q", stats.Product) @@ -48,7 +47,7 @@ func TestParseNvidiaSMILicensed(t *testing.T) { assert.Equal(t, "NVIDIA RTX Virtual Workstation", stats.LicensedProduct) assert.True(t, stats.Licensed) require.NotNil(t, stats.LicenseExpiry) - assert.Equal(t, 15*time.Minute, stats.LicenseExpiry.Sub(now)) + assert.Equal(t, time.Date(2026, 7, 7, 16, 1, 44, 0, time.UTC), stats.LicenseExpiry.UTC()) require.NotNil(t, stats.GPUUtil) assert.Equal(t, 7.0, *stats.GPUUtil) @@ -65,12 +64,23 @@ func TestParseNvidiaSMILicensed(t *testing.T) { func TestParseNvidiaSMIUnlicensed(t *testing.T) { xml := strings.Replace(smiLicensed, "Licensed (Expiry: 2026-7-7 16:1:44 GMT)", "Unlicensed (Restricted)", 1) - stats, err := parseNvidiaSMI([]byte(xml), time.Now()) + stats, err := parseNvidiaSMI([]byte(xml)) require.NoError(t, err) assert.False(t, stats.Licensed) assert.Nil(t, stats.LicenseExpiry) } +func TestParseNvidiaSMIUnknownZone(t *testing.T) { + // A non-GMT zone must fail loudly (no expiry) instead of silently + // parsing with a zero offset. + xml := strings.Replace(smiLicensed, + "Expiry: 2026-7-7 16:1:44 GMT", "Expiry: 2026-7-7 16:1:44 PST", 1) + stats, err := parseNvidiaSMI([]byte(xml)) + require.NoError(t, err) + assert.True(t, stats.Licensed) + assert.Nil(t, stats.LicenseExpiry) +} + func TestGPUCollectorNoGPU(t *testing.T) { c := NewGPUCollector() c.devicesDir = t.TempDir() // exists but empty diff --git a/server/lib/metrics/metrics.go b/server/lib/metrics/metrics.go index 3e58e406..545eead3 100644 --- a/server/lib/metrics/metrics.go +++ b/server/lib/metrics/metrics.go @@ -21,8 +21,9 @@ import ( type Collector interface { // Name identifies the collector in error logs. Name() string - // Collect appends samples to w. Returning an error does not fail the - // scrape; the other collectors' output is still served. + // Collect appends samples to w. On error the collector's entire + // output is discarded — partially written metric families never reach + // the scraper — and the other collectors' output is still served. Collect(ctx context.Context, w *Writer) error } @@ -66,6 +67,10 @@ func (w *Writer) Bytes() []byte { return w.buf.Bytes() } +func (w *Writer) append(other *Writer) { + w.buf.Write(other.buf.Bytes()) +} + func formatValue(v float64) string { if math.IsInf(v, 1) { return "+Inf" @@ -73,10 +78,18 @@ func formatValue(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) } -const scrapeTimeout = 10 * time.Second +const ( + scrapeTimeout = 10 * time.Second + // Each collector gets its own deadline under the scrape budget so a + // slow one (usually Chrome under load) cannot starve the others. + collectorTimeout = 4 * time.Second +) // Handler serves GET /metrics. Scrapes are serialized: concurrent requests -// wait rather than probing Chrome and nvidia-smi in parallel. +// wait rather than probing Chrome and nvidia-smi in parallel. Each +// collector writes into its own buffer that is appended only on success, +// so a failure never leaves a partially written metric family in the +// response. func Handler(log *slog.Logger, collectors ...Collector) http.Handler { var mu sync.Mutex return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { @@ -88,9 +101,15 @@ func Handler(log *slog.Logger, collectors ...Collector) http.Handler { w := &Writer{} for _, c := range collectors { - if err := c.Collect(ctx, w); err != nil { + cw := &Writer{} + cctx, ccancel := context.WithTimeout(ctx, collectorTimeout) + err := c.Collect(cctx, cw) + ccancel() + if err != nil { log.Error("metrics collector failed", "collector", c.Name(), "err", err) + continue } + w.append(cw) } rw.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") rw.Write(w.Bytes()) diff --git a/server/lib/metrics/metrics_test.go b/server/lib/metrics/metrics_test.go index e8594042..2a4179c8 100644 --- a/server/lib/metrics/metrics_test.go +++ b/server/lib/metrics/metrics_test.go @@ -37,10 +37,10 @@ func (s *stubCollector) Collect(_ context.Context, w *Writer) error { return s.fn(w) } -func TestHandlerServesAllCollectorsDespiteError(t *testing.T) { +func TestHandlerDiscardsFailedCollectorOutput(t *testing.T) { failing := &stubCollector{name: "bad", fn: func(w *Writer) error { - w.Metric("bad_up", "Whether bad works.", "gauge") - w.Sample("bad_up", nil, 0) + w.Metric("bad_partial", "A partially written family.", "histogram") + w.Sample("bad_partial_sum", nil, 1) return errors.New("boom") }} ok := &stubCollector{name: "good", fn: func(w *Writer) error { @@ -55,6 +55,6 @@ func TestHandlerServesAllCollectorsDespiteError(t *testing.T) { require.Equal(t, 200, rec.Code) assert.Equal(t, "text/plain; version=0.0.4; charset=utf-8", rec.Header().Get("Content-Type")) - assert.Contains(t, rec.Body.String(), "bad_up 0\n") + assert.NotContains(t, rec.Body.String(), "bad_partial") assert.Contains(t, rec.Body.String(), "good_total 3\n") } diff --git a/server/lib/metrics/system.go b/server/lib/metrics/system.go index 859b872b..0db76f3e 100644 --- a/server/lib/metrics/system.go +++ b/server/lib/metrics/system.go @@ -60,14 +60,33 @@ func (c *SystemCollector) Collect(ctx context.Context, w *Writer) error { if data, err := os.ReadFile(filepath.Join(c.procDir, "uptime")); err == nil { if fields := strings.Fields(string(data)); len(fields) > 0 { if v, err := strconv.ParseFloat(fields[0], 64); err == nil { - // CLOCK_MONOTONIC pauses while a scale-to-zero VM is - // suspended, so this measures active runtime, not wall age. + // /proc/uptime is CLOCK_BOOTTIME-based, but an externally + // paused VM's clocks stop entirely, so this still measures + // active runtime rather than wall age. w.Metric("kernel_vm_uptime_seconds", "VM active runtime; excludes time spent suspended.", "gauge") w.Sample("kernel_vm_uptime_seconds", nil, v) } } } + psi := false + for _, resource := range []string{"cpu", "memory", "io"} { + data, err := os.ReadFile(filepath.Join(c.procDir, "pressure", resource)) + if err != nil { + continue + } + v, ok := parsePressureSomeAvg10(string(data)) + if !ok { + continue + } + if !psi { + w.Metric("kernel_vm_pressure_some_avg10_percent", + "PSI: percentage of the last 10s in which at least one task stalled on the resource.", "gauge") + psi = true + } + w.Sample("kernel_vm_pressure_some_avg10_percent", []Label{{"resource", resource}}, v) + } + if data, err := os.ReadFile(filepath.Join(c.procDir, "net", "dev")); err == nil { rx, tx := parseNetDev(string(data)) w.Metric("kernel_vm_network_receive_bytes_total", "Bytes received on non-loopback interfaces.", "counter") @@ -169,8 +188,28 @@ func parseNetDev(data string) (rx, tx float64) { return rx, tx } +// parsePressureSomeAvg10 extracts the avg10 value from the "some" line of +// a /proc/pressure file, e.g. "some avg10=1.23 avg60=... total=...". +func parsePressureSomeAvg10(data string) (float64, bool) { + for _, line := range strings.Split(data, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "some" { + continue + } + for _, f := range fields[1:] { + if val, ok := strings.CutPrefix(f, "avg10="); ok { + v, err := strconv.ParseFloat(val, 64) + return v, err == nil + } + } + } + return 0, false +} + // chromiumProcesses counts processes whose comm is "chromium" and sums -// their resident memory. +// their resident memory. Summing per-process RSS double-counts shared +// pages, so treat the total as an upper bound useful for trends rather +// than absolute usage. func (c *SystemCollector) chromiumProcesses() (count int, rssBytes float64) { entries, err := os.ReadDir(c.procDir) if err != nil { diff --git a/server/lib/metrics/system_test.go b/server/lib/metrics/system_test.go index 15eb9b9f..72676c23 100644 --- a/server/lib/metrics/system_test.go +++ b/server/lib/metrics/system_test.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "os" "path/filepath" "testing" @@ -44,6 +45,33 @@ func TestParseNetDev(t *testing.T) { assert.Equal(t, 500000.0, tx) } +func TestParsePressureSomeAvg10(t *testing.T) { + v, ok := parsePressureSomeAvg10("some avg10=1.53 avg60=0.42 avg300=0.11 total=123456\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n") + require.True(t, ok) + assert.Equal(t, 1.53, v) + + _, ok = parsePressureSomeAvg10("garbage\n") + assert.False(t, ok) +} + +func TestPSIEmission(t *testing.T) { + proc := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(proc, "pressure"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(proc, "pressure", "cpu"), + []byte("some avg10=1.53 avg60=0.42 avg300=0.11 total=123\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(proc, "pressure", "io"), + []byte("some avg10=0.07 avg60=0.01 avg300=0.00 total=9\n"), 0o644)) + // memory intentionally absent: emission must skip it without failing. + + c := &SystemCollector{procDir: proc, rootPath: "/"} + w := &Writer{} + require.NoError(t, c.Collect(context.Background(), w)) + out := string(w.Bytes()) + assert.Contains(t, out, `kernel_vm_pressure_some_avg10_percent{resource="cpu"} 1.53`) + assert.Contains(t, out, `kernel_vm_pressure_some_avg10_percent{resource="io"} 0.07`) + assert.NotContains(t, out, `resource="memory"`) +} + func TestChromiumProcesses(t *testing.T) { proc := t.TempDir() mkProc := func(pid, comm, statm string) {