diff --git a/.changeset/admin-dashboard-ui.md b/.changeset/admin-dashboard-ui.md new file mode 100644 index 00000000..c6c8b2e0 --- /dev/null +++ b/.changeset/admin-dashboard-ui.md @@ -0,0 +1,5 @@ +--- +"nostream": minor +--- + +feat: add admin observability dashboard with Grafana embed and provisioned metrics panels diff --git a/.changeset/admin-metrics-sse-endpoint.md b/.changeset/admin-metrics-sse-endpoint.md new file mode 100644 index 00000000..2431746d --- /dev/null +++ b/.changeset/admin-metrics-sse-endpoint.md @@ -0,0 +1,5 @@ +--- +"nostream": minor +--- + +feat: add Prometheus-backed admin metrics SSE endpoint diff --git a/.env.example b/.env.example index 66c82e1d..509e2f02 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,21 @@ WORKER_COUNT=2 # Defaults to CPU count. Use 1 or 2 for local testing. # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 # OTEL_METRIC_EXPORT_INTERVAL_MS=10000 +# --- OBSERVABILITY (Prometheus for Admin Metrics) --- +# PROMETHEUS_URL=http://127.0.0.1:9090 +# PROMETHEUS_QUERY_TIMEOUT_MS=5000 +# ADMIN_METRICS_SSE_INTERVAL_MS=5000 +# ADMIN_METRICS_SNAPSHOT_TIMEOUT_MS=10000 +# ADMIN_DEPENDENCY_PING_TIMEOUT_MS=3000 + +# --- ADMIN CONSOLE (opt-in; disabled by default in default-settings.yaml) --- +# ADMIN_PASSWORD=dev-admin-password +# Set admin.enabled: true in .nostr/settings.yaml to enable the console +# GRAFANA_URL=http://127.0.0.1:3000 +# GRAFANA_DASHBOARD_UID=nostream-overview +# GRAFANA_ADMIN_USER=admin +# GRAFANA_ADMIN_PASSWORD=admin + # --- RELAY PRIVATE KEY (Optional) --- # RELAY_PRIVATE_KEY=your_hex_private_key diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 473edb9b..ad47ce3a 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -54,6 +54,11 @@ The following environment variables can be set: | REDIS_PORT | Redis Port | 6379 | | REDIS_USER | Redis User | default | | REDIS_PASSWORD | Redis Password | nostr_ts_relay | +| PROMETHEUS_URL | Prometheus base URL for admin metrics queries | http://127.0.0.1:9090 | +| PROMETHEUS_QUERY_TIMEOUT_MS | Timeout for each Prometheus admin metrics query (ms) | 5000 | +| ADMIN_METRICS_SSE_INTERVAL_MS | Interval between `/admin/metrics` SSE snapshots (ms) | 5000 | +| ADMIN_METRICS_SNAPSHOT_TIMEOUT_MS| Timeout for collecting one `/admin/metrics` snapshot (ms) | 10000 | +| ADMIN_DEPENDENCY_PING_TIMEOUT_MS | Timeout for admin DB/Redis dependency pings (ms) | 3000 | | NOSTR_CONFIG_DIR | Configuration directory | /.nostr/ | | DEBUG | Debugging filter | | | ZEBEDEE_API_KEY | Zebedee Project API Key | | diff --git a/docker-compose.yml b/docker-compose.yml index 294cc173..1939bd63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,8 @@ services: OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-nostream} OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} OTEL_METRIC_EXPORT_INTERVAL_MS: ${OTEL_METRIC_EXPORT_INTERVAL_MS:-10000} + PROMETHEUS_URL: ${PROMETHEUS_URL:-http://prometheus:9090} + ADMIN_METRICS_SSE_INTERVAL_MS: ${ADMIN_METRICS_SSE_INTERVAL_MS:-5000} # Enable DEBUG for troubleshooting. Examples: # DEBUG: "primary:*" # DEBUG: "worker:*" @@ -177,6 +179,29 @@ services: networks: default: + grafana: + image: grafana/grafana:11.2.0 + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_AUTH_ANONYMOUS_ENABLED: 'true' + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SECURITY_ALLOW_EMBEDDING: 'true' + GF_SERVER_ROOT_URL: ${GRAFANA_URL:-http://127.0.0.1:3000} + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: + - 127.0.0.1:3000:3000 + depends_on: + prometheus: + condition: service_started + restart: always + networks: + default: + networks: default: name: nostream @@ -188,3 +213,4 @@ networks: volumes: cache: prometheus-data: + grafana-data: diff --git a/docs/REDIS.md b/docs/REDIS.md index f3dfdcf8..3b08ebd5 100644 --- a/docs/REDIS.md +++ b/docs/REDIS.md @@ -11,6 +11,8 @@ Nostream needs. Rate limiting is implemented using a sliding window strategy, which uses Redis sorted sets to track request timestamps. This allows Nostream to accurately enforce rate limits over a rolling time window rather than a fixed one, preventing clients from bursting requests at window boundaries. +When Redis is temporarily unavailable, Nostream keeps serving the relay and admin HTTP endpoints and treats rate-limiter Redis failures as fail-open (events are not blocked solely because Redis is down). Admin health degrades accordingly (`redis.ok=false`), so operators can detect and recover Redis without taking the relay offline. + ## Requirements - Redis 6.0 or higher (for ACL support) diff --git a/grafana/dashboards/nostream-overview.json b/grafana/dashboards/nostream-overview.json new file mode 100644 index 00000000..6210c65e --- /dev/null +++ b/grafana/dashboards/nostream-overview.json @@ -0,0 +1,763 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(rate(nostream_events_accepted_total[1m]))", + "refId": "A" + } + ], + "title": "Events / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(rate(nostream_events_rejected_total[1m]))", + "refId": "A" + } + ], + "title": "Rejected / sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(nostream_websocket_connections)", + "refId": "A" + } + ], + "title": "Active connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "avg(nostream_process_cpu_load_percent)", + "refId": "A" + } + ], + "title": "CPU load %", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(nostream_events_accepted_total)", + "refId": "A" + } + ], + "title": "Accepted total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(nostream_events_rejected_total)", + "refId": "A" + } + ], + "title": "Rejected total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rejected" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(rate(nostream_events_accepted_total[1m]))", + "legendFormat": "accepted", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(rate(nostream_events_rejected_total[1m]))", + "legendFormat": "rejected", + "refId": "B" + } + ], + "title": "Events / sec (15m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum(nostream_websocket_connections)", + "legendFormat": "connections", + "refId": "A" + } + ], + "title": "WebSocket connections (15m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "sum by (worker_type) (nostream_process_memory_used_mb)", + "legendFormat": "{{worker_type}}", + "refId": "A" + } + ], + "title": "Memory by worker", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "expr": "avg by (worker_type) (nostream_process_cpu_load_percent)", + "legendFormat": "{{worker_type}}", + "refId": "A" + } + ], + "title": "CPU by worker", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["nostream", "admin"], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Nostream Overview", + "uid": "nostream-overview", + "version": 1, + "weekStart": "" +} diff --git a/grafana/provisioning/dashboards/default.yaml b/grafana/provisioning/dashboards/default.yaml new file mode 100644 index 00000000..022ad542 --- /dev/null +++ b/grafana/provisioning/dashboards/default.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: nostream + orgId: 1 + folder: Nostream + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/grafana/provisioning/datasources/prometheus.yaml b/grafana/provisioning/datasources/prometheus.yaml new file mode 100644 index 00000000..9e3c7b4e --- /dev/null +++ b/grafana/provisioning/datasources/prometheus.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/resources/admin/assets/dashboard.css b/resources/admin/assets/dashboard.css new file mode 100644 index 00000000..f6aaa594 --- /dev/null +++ b/resources/admin/assets/dashboard.css @@ -0,0 +1,447 @@ +.admin-dashboard.console-theme { + --bg: #ece6dc; + --panel: #f6f2ec; + --panel-border: #b8aa98; + --text: #2c241c; + --text-muted: #6b5f52; + --label: #5a4f44; + --accent: #6b5344; + --prompt: #8a5a12; + --ok: #2f6b4f; + --warn: #8a5a12; + --danger: #8f3a3a; + --no-data: #5a6270; + min-height: 100vh; + background: var(--bg); + color: var(--text); + font-family: "IBM Plex Mono", "Courier New", monospace; + font-size: 0.875rem; +} + +.admin-dashboard.console-theme[data-theme="dark"] { + --bg: #0e120e; + --panel: #141a14; + --panel-border: #2e3e2e; + --text: #b8d8b8; + --text-muted: #6a906a; + --label: #8cb08c; + --accent: #7ecf7e; + --prompt: #9ecf6a; + --ok: #6ecf8a; + --warn: #d4b04a; + --danger: #e07a7a; + --no-data: #8aa4c8; + background-color: var(--bg); + background-image: repeating-linear-gradient( + 0deg, + transparent, + transparent 2px, + rgba(110, 207, 138, 0.02) 2px, + rgba(110, 207, 138, 0.02) 3px + ); +} + +.prompt-char { + color: var(--prompt); + font-weight: 700; + margin-right: 0.25rem; +} + +.module-code { + color: var(--text-muted); + font-size: 0.62rem; + font-weight: 600; + letter-spacing: 0.06em; + margin-right: 0.35rem; +} + +.admin-header { + border-bottom: 1px solid var(--panel-border); + padding-bottom: 1rem; +} + +.admin-dashboard .admin-kicker { + color: var(--text-muted); + font-size: 0.7rem; + font-weight: 500; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.admin-dashboard .admin-title { + color: var(--text); + font-size: 1.1rem; + font-weight: 600; +} + +.admin-dashboard .admin-muted { + color: var(--text-muted); +} + +.terminal-line { + color: var(--text-muted); + font-size: 0.75rem; +} + +.admin-dashboard .field-label, +.admin-dashboard .metric-label { + color: var(--label); + font-size: 0.7rem; + font-weight: 500; + letter-spacing: 0.04em; + margin-bottom: 0.35rem; + text-transform: uppercase; +} + +.clock-wrap { + border: 1px solid var(--panel-border); + padding: 0.25rem 0.5rem; + text-align: right; +} + +.clock-value { + color: var(--text); + font-size: 0.9rem; + font-variant-numeric: tabular-nums; + font-weight: 600; + line-height: 1.1; +} + +.clock-label { + color: var(--text-muted); + font-size: 0.65rem; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.system-alert { + border: 1px solid var(--warn); + color: var(--warn); + font-size: 0.75rem; + margin-bottom: 0.75rem; + padding: 0.45rem 0.75rem; +} + +.system-alert.degraded { + background: color-mix(in srgb, var(--warn) 10%, var(--panel)); + border-color: var(--warn); + color: var(--warn); +} + +.system-alert.unavailable { + background: color-mix(in srgb, var(--danger) 10%, var(--panel)); + border-color: var(--danger); + color: var(--danger); +} + +.dashboard-section { + margin-bottom: 1.5rem; +} + +.section-title::before { + color: var(--prompt); + content: "> "; + font-weight: 700; +} + +.section-title { + border-bottom: 1px solid var(--panel-border); + color: var(--label); + font-size: 0.75rem; + font-weight: 600; + letter-spacing: 0.08em; + margin-bottom: 0.75rem; + padding-bottom: 0.35rem; + text-transform: uppercase; +} + +.panel-card, +.metric-card { + background: var(--panel); + border: 1px solid var(--panel-border); + padding: 0.85rem 1rem; + transition: border-color 180ms ease, background-color 180ms ease; +} + +.metric-card.updated { + border-color: var(--accent); +} + +.panel-title { + color: var(--text); + font-size: 0.95rem; + font-weight: 600; +} + +.metric-card { + min-height: 4.75rem; +} + +.status-led { + background: var(--no-data); + border-radius: 50%; + display: inline-block; + height: 0.45rem; + margin-right: 0.4rem; + vertical-align: middle; + width: 0.45rem; +} + +.status-ok .status-led, +.status-badge.live .status-led { + animation: led-pulse 2s ease-in-out infinite; + background: var(--ok); + box-shadow: 0 0 4px color-mix(in srgb, var(--ok) 60%, transparent); +} + +.status-degraded .status-led, +.status-badge.warn .status-led { + background: var(--warn); +} + +.status-unavailable .status-led, +.status-down .status-led, +.status-badge.error .status-led { + background: var(--danger); +} + +.status-no-data .status-led, +.status-badge:not(.live):not(.warn):not(.error) .status-led { + background: var(--no-data); +} + +@keyframes led-pulse { + 0%, + 100% { + opacity: 1; + } + + 50% { + opacity: 0.55; + } +} + +.admin-dashboard .metric-value { + color: var(--text); + font-size: 1.15rem; + font-variant-numeric: tabular-nums; + font-weight: 600; + line-height: 1.3; +} + +.admin-dashboard .metric-number { + font-size: inherit; +} + +.admin-dashboard .metric-unit { + color: var(--text-muted); + font-size: 0.8rem; + font-weight: 400; +} + +.admin-dashboard .status-ok { + color: var(--ok); +} + +.admin-dashboard .status-degraded { + color: var(--warn); +} + +.admin-dashboard .status-unavailable, +.admin-dashboard .status-down { + color: var(--danger); +} + +.admin-dashboard .status-no-data { + color: var(--no-data); +} + +.sync-line { + color: var(--text-muted); + font-size: 0.75rem; + font-variant-numeric: tabular-nums; +} + +.sync-cursor { + animation: cursor-blink 1s step-end infinite; + background: var(--accent); + display: inline-block; + height: 0.85em; + margin-left: 0.15rem; + vertical-align: text-bottom; + width: 0.45rem; +} + +@keyframes cursor-blink { + 0%, + 100% { + opacity: 1; + } + + 50% { + opacity: 0; + } +} + +.status-badge { + align-items: center; + border: 1px solid var(--panel-border); + color: var(--text-muted); + display: inline-flex; + font-size: 0.68rem; + font-weight: 500; + gap: 0.15rem; + letter-spacing: 0.04em; + padding: 0.25rem 0.55rem; + text-transform: uppercase; +} + +.status-badge.live { + border-color: var(--ok); + color: var(--ok); +} + +.status-badge.warn { + border-color: var(--warn); + color: var(--warn); +} + +.status-badge.error { + border-color: var(--danger); + color: var(--danger); +} + +.admin-dashboard .btn-console { + background: var(--panel); + border: 1px solid var(--panel-border); + color: var(--text); + font-family: inherit; + font-size: 0.75rem; + font-weight: 500; +} + +.admin-dashboard .btn-console:hover, +.admin-dashboard .btn-console:focus { + background: color-mix(in srgb, var(--accent) 8%, var(--panel)); + border-color: var(--accent); + color: var(--text); +} + +.admin-dashboard .btn-console.copied { + border-color: var(--ok); + color: var(--ok); +} + +.admin-dashboard .console-input { + background: var(--bg); + border: 1px solid var(--panel-border); + color: var(--text); + font-family: inherit; + font-size: 0.875rem; +} + +.admin-dashboard .console-input:focus { + background: var(--bg); + border-color: var(--accent); + box-shadow: 0 0 0 2px color-mix(in srgb, var(--accent) 20%, transparent); + color: var(--text); +} + +.admin-dashboard .alert-danger { + background: color-mix(in srgb, var(--danger) 8%, var(--panel)); + border: 1px solid color-mix(in srgb, var(--danger) 40%, var(--panel-border)); + color: var(--danger); + font-size: 0.8rem; +} + +.admin-dashboard .login-box { + margin: 2rem auto 0; + max-width: 28rem; +} + +.dashboard-nav { + background: var(--panel); + border: 1px solid var(--panel-border); + left: 0.75rem; + min-width: 12rem; + padding: 0.85rem; + position: fixed; + top: 4.5rem; + transform: translateX(-120%); + transition: transform 160ms ease; + z-index: 1041; +} + +.admin-dashboard.nav-open .dashboard-nav { + transform: translateX(0); +} + +.nav-item { + background: var(--bg); + border: 1px solid var(--panel-border); + color: var(--text); + display: block; + font-family: inherit; + font-size: 0.75rem; + font-weight: 500; + margin-bottom: 0.4rem; + padding: 0.45rem 0.55rem; + text-align: left; + width: 100%; +} + +.nav-item.active { + border-color: var(--accent); + color: var(--accent); +} + +.nav-overlay { + background: rgba(0, 0, 0, 0.25); + bottom: 0; + display: none; + left: 0; + position: fixed; + right: 0; + top: 0; + z-index: 1040; +} + +.admin-dashboard.nav-open .nav-overlay { + display: block; +} + +.chart-panel { + padding: 0; +} + +.panel-header { + border-bottom: 1px solid var(--panel-border); + color: var(--label); + font-size: 0.7rem; + font-weight: 500; + letter-spacing: 0.04em; + padding: 0.55rem 0.85rem; + text-transform: uppercase; +} + +.admin-dashboard .grafana-frame { + background: #141814; + border: 0; + display: block; + height: 380px; + width: 100%; +} + +.admin-dashboard[data-theme="light"] .grafana-frame { + background: #1a1814; +} + +@media (max-width: 768px) { + .admin-dashboard .metric-value { + font-size: 1rem; + } + + .admin-dashboard .grafana-frame { + height: 300px; + } +} diff --git a/resources/admin/assets/dashboard.js b/resources/admin/assets/dashboard.js new file mode 100644 index 00000000..38a979b4 --- /dev/null +++ b/resources/admin/assets/dashboard.js @@ -0,0 +1,612 @@ +(() => { + const config = window.__ADMIN_DASHBOARD__ || { pathPrefix: '' } + const pathPrefix = config.pathPrefix || '' + const adminBase = `${pathPrefix}/admin` + + const loginPanel = document.getElementById('login-panel') + const dashboardPanel = document.getElementById('dashboard-panel') + const dashboardActions = document.getElementById('dashboard-actions') + const loginForm = document.getElementById('login-form') + const loginError = document.getElementById('login-error') + const loginButton = document.getElementById('login-button') + const logoutButton = document.getElementById('logout-button') + const copySnapshotButton = document.getElementById('copy-snapshot-button') + const streamStatus = document.getElementById('stream-status') + const passwordInput = document.getElementById('password') + const themeToggle = document.getElementById('theme-toggle') + const liveClock = document.getElementById('live-clock') + const menuToggle = document.getElementById('menu-toggle') + const navOverlay = document.getElementById('nav-overlay') + const navPanel = document.getElementById('dashboard-nav') + const navItems = document.querySelectorAll('.nav-item') + const systemAlert = document.getElementById('system-alert') + + let metricsSource + let reconnectTimer + let reconnectAttempt = 0 + let streamClosedIntentionally = false + let lastSnapshot + let relativeTimeTimer + let staleCheckTimer + const staleThresholdMs = 15000 + + const statusClasses = ['status-ok', 'status-degraded', 'status-unavailable', 'status-down', 'status-no-data'] + + const statusLabels = { + system: { + ok: '[OK]', + degraded: '[WARN]', + unavailable: '[NULL]', + }, + prometheus: { + online: '[OK]', + noData: '[NULL]', + }, + database: { + healthy: '[OK]', + fault: '[FAULT]', + }, + redis: { + healthy: '[OK]', + fault: '[FAULT]', + }, + stream: { + offline: '[OFF]', + init: '[SYNC]', + live: '[LIVE]', + stale: '[STALE]', + parseError: '[ERR]', + reconnect: '[RETRY]', + }, + } + + const getTheme = () => { + const stored = localStorage.getItem('admin_theme') + if (stored === 'light' || stored === 'dark') { + return stored + } + + return 'light' + } + + const applyTheme = (theme) => { + document.body.setAttribute('data-theme', theme) + if (themeToggle) { + themeToggle.textContent = theme === 'dark' ? 'Dark' : 'Light' + } + } + + const formatClock = () => { + return new Date().toISOString().slice(11, 19) + } + + const formatRelativeTime = (timestampMs) => { + const seconds = Math.max(0, Math.floor((Date.now() - timestampMs) / 1000)) + if (seconds < 8) { + return 'just now' + } + if (seconds < 60) { + return `${seconds}s ago` + } + if (seconds < 3600) { + return `${Math.floor(seconds / 60)}m ago` + } + + return `${Math.floor(seconds / 3600)}h ago` + } + + const formatTimestamp = (timestampMs) => { + const updated = new Date(timestampMs).toISOString().replace('T', ' ').replace(/\.\d{3}Z$/, ' UTC') + return `> sync: ${updated} (${formatRelativeTime(timestampMs)})` + } + + const sumMetricValues = (...values) => { + return values.reduce((total, value) => { + const parsed = Number(value) + return total + (Number.isFinite(parsed) ? parsed : 0) + }, 0) + } + + const formatNumber = (value, digits = 0) => { + const parsed = Number(value) + if (!Number.isFinite(parsed)) { + return '0' + } + + return parsed.toLocaleString(undefined, { + maximumFractionDigits: digits, + minimumFractionDigits: digits, + }) + } + + const setText = (id, value) => { + const element = document.getElementById(id) + if (element) { + element.textContent = value + } + } + + const setSyncLine = (timestampMs) => { + const element = document.getElementById('metric-updated-at') + if (!element) { + return + } + + element.innerHTML = `${formatTimestamp(timestampMs)}` + } + + const setMetricValue = (id, value, digits = 0) => { + const element = document.getElementById(id) + if (!element) { + return + } + + const numberNode = element.querySelector('.metric-number') + if (numberNode) { + numberNode.textContent = formatNumber(value, digits) + return + } + + element.textContent = formatNumber(value, digits) + } + + const setStatusText = (id, value, statusClass) => { + const element = document.getElementById(id) + if (!element) { + return + } + + element.classList.remove(...statusClasses) + if (statusClass) { + element.classList.add(statusClass) + } + + const led = element.querySelector('.status-led') + element.replaceChildren() + if (led) { + element.appendChild(led) + } + element.appendChild(document.createTextNode(value)) + } + + const setStreamStatus = (label, statusClass) => { + if (!streamStatus) { + return + } + + streamStatus.className = `status-badge${statusClass ? ` ${statusClass}` : ''}` + streamStatus.replaceChildren() + + const led = document.createElement('span') + led.className = 'status-led' + led.setAttribute('aria-hidden', 'true') + streamStatus.appendChild(led) + streamStatus.appendChild(document.createTextNode(label)) + } + + const updateSystemAlert = (snapshot) => { + if (!systemAlert) { + return + } + + if (lastSnapshot?.timestamp && Date.now() - lastSnapshot.timestamp > staleThresholdMs) { + systemAlert.textContent = '[WARN] Metrics stale — waiting for backend update.' + systemAlert.className = 'system-alert degraded' + return + } + + if (snapshot.status === 'degraded') { + systemAlert.textContent = '[WARN] System degraded — check dependency health below.' + systemAlert.className = 'system-alert degraded' + return + } + + if (snapshot.status === 'unavailable') { + systemAlert.textContent = '[NULL] Metrics unavailable — Prometheus returned no data.' + systemAlert.className = 'system-alert unavailable' + return + } + + systemAlert.className = 'system-alert d-none' + systemAlert.textContent = '' + } + + const scheduleRelativeTimeRefresh = (timestampMs) => { + if (relativeTimeTimer) { + clearInterval(relativeTimeTimer) + } + + relativeTimeTimer = setInterval(() => { + if (lastSnapshot?.timestamp) { + setSyncLine(lastSnapshot.timestamp) + checkStaleSnapshot() + } + }, 10000) + } + + const checkStaleSnapshot = () => { + if (streamClosedIntentionally || !lastSnapshot?.timestamp) { + return + } + + const isStale = Date.now() - lastSnapshot.timestamp > staleThresholdMs + if (isStale) { + setStreamStatus(statusLabels.stream.stale, 'warn') + updateSystemAlert(lastSnapshot) + return + } + + setStreamStatus(statusLabels.stream.live, 'live') + updateSystemAlert(lastSnapshot) + } + + const startStaleCheck = () => { + if (staleCheckTimer) { + clearInterval(staleCheckTimer) + } + + staleCheckTimer = setInterval(checkStaleSnapshot, 3000) + } + + const stopStaleCheck = () => { + if (staleCheckTimer) { + clearInterval(staleCheckTimer) + staleCheckTimer = undefined + } + } + + const setNavOpen = (isOpen) => { + document.body.classList.toggle('nav-open', isOpen) + if (menuToggle) { + menuToggle.setAttribute('aria-expanded', String(isOpen)) + } + if (navPanel) { + navPanel.setAttribute('aria-hidden', String(!isOpen)) + } + } + + const isTypingTarget = (target) => { + if (!(target instanceof HTMLElement)) { + return false + } + + const tag = target.tagName.toLowerCase() + return tag === 'input' || tag === 'textarea' || target.isContentEditable + } + + const showLogin = () => { + stopMetricsStream() + setNavOpen(false) + loginPanel.classList.remove('d-none') + dashboardPanel.classList.add('d-none') + dashboardActions.classList.add('d-none') + dashboardActions.classList.remove('d-flex') + menuToggle?.classList.add('d-none') + loginError.classList.add('d-none') + passwordInput.value = '' + lastSnapshot = undefined + } + + const showDashboard = () => { + loginPanel.classList.add('d-none') + dashboardPanel.classList.remove('d-none') + dashboardActions.classList.remove('d-none') + dashboardActions.classList.add('d-flex') + menuToggle?.classList.remove('d-none') + + document.querySelectorAll('.grafana-frame').forEach((frame) => { + if (!frame.src && frame.dataset.panelId) { + const theme = document.body.getAttribute('data-theme') === 'dark' ? 'dark' : 'light' + const baseUrl = (config.grafanaBaseUrl || config.grafanaUrl || 'http://127.0.0.1:3000').replace(/\/+$/, '') + const dashboardUid = config.grafanaDashboardUid || 'nostream-overview' + const params = new URLSearchParams({ + orgId: '1', + panelId: frame.dataset.panelId, + refresh: '5s', + theme, + }) + frame.src = `${baseUrl}/d-solo/${dashboardUid}?${params.toString()}` + } + }) + + startMetricsStream() + } + + const showLoginError = (message) => { + loginError.textContent = message + loginError.classList.remove('d-none') + } + + const refreshGrafanaFrames = () => { + const currentTheme = document.body.getAttribute('data-theme') === 'dark' ? 'dark' : 'light' + const baseUrl = (config.grafanaBaseUrl || config.grafanaUrl || 'http://127.0.0.1:3000').replace(/\/+$/, '') + const dashboardUid = config.grafanaDashboardUid || 'nostream-overview' + + document.querySelectorAll('.grafana-frame').forEach((frame) => { + if (!frame.src || !frame.dataset.panelId) { + return + } + + const params = new URLSearchParams({ + orgId: '1', + panelId: frame.dataset.panelId, + refresh: '5s', + theme: currentTheme, + }) + frame.src = `${baseUrl}/d-solo/${dashboardUid}?${params.toString()}` + }) + } + + const updateSnapshot = (snapshot) => { + lastSnapshot = snapshot + + const statusClass = + snapshot.status === 'ok' + ? 'status-ok' + : snapshot.status === 'degraded' + ? 'status-degraded' + : 'status-unavailable' + + setStatusText('metric-status', statusLabels.system[snapshot.status] ?? statusLabels.system.unavailable, statusClass) + setStatusText( + 'metric-prometheus', + snapshot.prometheus?.available ? statusLabels.prometheus.online : statusLabels.prometheus.noData, + snapshot.prometheus?.available ? 'status-ok' : 'status-no-data', + ) + setStatusText( + 'metric-database', + snapshot.health?.database?.ok ? statusLabels.database.healthy : statusLabels.database.fault, + snapshot.health?.database?.ok ? 'status-ok' : 'status-down', + ) + setStatusText( + 'metric-redis', + snapshot.health?.redis?.ok ? statusLabels.redis.healthy : statusLabels.redis.fault, + snapshot.health?.redis?.ok ? 'status-ok' : 'status-down', + ) + + setMetricValue('metric-events-per-second', snapshot.metrics?.eventsPerSecond, 2) + setMetricValue('metric-events-rejected-per-second', snapshot.metrics?.eventsRejectedPerSecond, 2) + setMetricValue('metric-active-connections', snapshot.metrics?.activeConnections) + setMetricValue('metric-cpu-load', snapshot.metrics?.cpuLoadPercent, 1) + setMetricValue( + 'metric-events-total', + sumMetricValues(snapshot.metrics?.eventsAcceptedTotal, snapshot.metrics?.eventsRejectedTotal), + ) + setMetricValue('metric-accepted-total', snapshot.metrics?.eventsAcceptedTotal) + setMetricValue('metric-rejected-total', snapshot.metrics?.eventsRejectedTotal) + setMetricValue('metric-memory-used', snapshot.metrics?.memoryUsedMb, 1) + + setSyncLine(snapshot.timestamp) + scheduleRelativeTimeRefresh(snapshot.timestamp) + updateSystemAlert(snapshot) + } + + const clearReconnectTimer = () => { + if (reconnectTimer) { + clearTimeout(reconnectTimer) + reconnectTimer = undefined + } + } + + const stopMetricsStream = () => { + streamClosedIntentionally = true + clearReconnectTimer() + if (relativeTimeTimer) { + clearInterval(relativeTimeTimer) + relativeTimeTimer = undefined + } + stopStaleCheck() + + if (metricsSource) { + metricsSource.close() + metricsSource = undefined + } + + setStreamStatus(statusLabels.stream.offline) + } + + const startMetricsStream = () => { + streamClosedIntentionally = false + stopMetricsStream() + streamClosedIntentionally = false + setStreamStatus(statusLabels.stream.init) + + metricsSource = new EventSource(`${adminBase}/metrics`, { withCredentials: true }) + + metricsSource.onopen = () => { + reconnectAttempt = 0 + setStreamStatus(statusLabels.stream.live, 'live') + startStaleCheck() + } + + metricsSource.onmessage = (event) => { + try { + updateSnapshot(JSON.parse(event.data)) + checkStaleSnapshot() + } catch { + setStreamStatus(statusLabels.stream.parseError, 'error') + } + } + + metricsSource.onerror = () => { + if (streamClosedIntentionally) { + return + } + + if (metricsSource) { + metricsSource.close() + metricsSource = undefined + } + + setStreamStatus(statusLabels.stream.reconnect, 'warn') + + const delayMs = Math.min(30000, 1000 * 2 ** reconnectAttempt) + reconnectAttempt += 1 + clearReconnectTimer() + reconnectTimer = setTimeout(() => { + startMetricsStream() + }, delayMs) + } + } + + const copyLatestSnapshot = async () => { + if (!copySnapshotButton || !lastSnapshot) { + return + } + + try { + await navigator.clipboard.writeText(JSON.stringify(lastSnapshot, null, 2)) + const original = copySnapshotButton.textContent + copySnapshotButton.textContent = 'Copied' + copySnapshotButton.classList.add('copied') + setTimeout(() => { + copySnapshotButton.textContent = original + copySnapshotButton.classList.remove('copied') + }, 2000) + } catch { + copySnapshotButton.textContent = 'Failed' + setTimeout(() => { + copySnapshotButton.textContent = 'Copy JSON' + }, 2000) + } + } + + const checkSession = async () => { + const response = await fetch(`${adminBase}/session`, { + credentials: 'include', + }) + + if (response.status === 401) { + showLogin() + return + } + + if (!response.ok) { + showLoginError('Unable to verify session.') + showLogin() + return + } + + const body = await response.json() + if (body.authenticated) { + showDashboard() + return + } + + showLogin() + } + + loginForm.addEventListener('submit', async (event) => { + event.preventDefault() + loginError.classList.add('d-none') + loginButton.disabled = true + + try { + const response = await fetch(`${adminBase}/login`, { + method: 'POST', + credentials: 'include', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify({ + password: passwordInput.value, + }), + }) + + if (response.status === 401) { + showLoginError('Invalid password.') + return + } + + if (!response.ok) { + showLoginError('Sign in failed.') + return + } + + showDashboard() + } catch { + showLoginError('Network error. Try again.') + } finally { + loginButton.disabled = false + } + }) + + logoutButton.addEventListener('click', async () => { + try { + await fetch(`${adminBase}/logout`, { + method: 'POST', + credentials: 'include', + }) + } catch { + // Still return to the login screen if the network request fails. + } + + showLogin() + }) + + copySnapshotButton?.addEventListener('click', () => { + void copyLatestSnapshot() + }) + + if (menuToggle) { + menuToggle.addEventListener('click', () => { + const isOpen = document.body.classList.contains('nav-open') + setNavOpen(!isOpen) + }) + } + + navOverlay?.addEventListener('click', () => { + setNavOpen(false) + }) + + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + setNavOpen(false) + return + } + + if (isTypingTarget(event.target)) { + return + } + + if (event.key === 't' || event.key === 'T') { + const current = document.body.getAttribute('data-theme') === 'dark' ? 'dark' : 'light' + const nextTheme = current === 'dark' ? 'light' : 'dark' + localStorage.setItem('admin_theme', nextTheme) + applyTheme(nextTheme) + refreshGrafanaFrames() + } + + if ((event.key === 'l' || event.key === 'L') && !dashboardPanel.classList.contains('d-none')) { + logoutButton?.click() + } + }) + + navItems.forEach((item) => { + item.addEventListener('click', () => { + navItems.forEach((candidate) => candidate.classList.remove('active')) + item.classList.add('active') + setNavOpen(false) + }) + }) + + if (themeToggle) { + themeToggle.addEventListener('click', () => { + const current = document.body.getAttribute('data-theme') === 'dark' ? 'dark' : 'light' + const nextTheme = current === 'dark' ? 'light' : 'dark' + localStorage.setItem('admin_theme', nextTheme) + applyTheme(nextTheme) + refreshGrafanaFrames() + }) + } + + if (liveClock) { + liveClock.textContent = formatClock() + setInterval(() => { + liveClock.textContent = formatClock() + }, 1000) + } + + applyTheme(getTheme()) + checkSession() +})() diff --git a/resources/admin/dashboard.html b/resources/admin/dashboard.html new file mode 100644 index 00000000..31a85ace --- /dev/null +++ b/resources/admin/dashboard.html @@ -0,0 +1,207 @@ + + + + + + {{name}} Admin + + + + + + + +
+
+
+ +
+

> Admin Console

+

{{name}}

+
+
+
+ +
+

--:--:--

+

UTC

+
+
+
+ Connecting + + Grafana + +
+
+ +
+
+ +
+
+ +
+ + + +
+ +
+
+

Health

+
+
+
+

SYS System

+

Pending

+
+
+
+
+

PROM Prometheus

+

Pending

+
+
+
+
+

DB Database

+

Pending

+
+
+
+
+

RDS Redis

+

Pending

+
+
+
+
+ +
+

Throughput

+
+
+
+

IN Events / sec

+

0 /s

+
+
+
+
+

RJ Rejected / sec

+

0 /s

+
+
+
+
+

WS Connections

+

0

+
+
+
+
+ +
+

Events

+
+
+
+

SUM Total

+

0

+
+
+
+
+

OK Accepted

+

0

+
+
+
+
+

ERR Rejected

+

0

+
+
+
+
+ +
+

Resources

+
+
+
+

CPU Load

+

0 %

+
+
+
+
+

MEM Used

+

0 MB

+
+
+
+
+ +

> sync: waiting for metrics…

+ +
+

Charts

+
+
+
+
CHT Events per second (15m)
+ +
+
+
+
+
CHT WebSocket connections (15m)
+ +
+
+
+
+
CHT Memory by worker
+ +
+
+
+
+
CHT CPU by worker
+ +
+
+
+
+
+
+
+ + + + + diff --git a/src/@types/admin.ts b/src/@types/admin.ts index 955e185f..0a742db1 100644 --- a/src/@types/admin.ts +++ b/src/@types/admin.ts @@ -2,6 +2,7 @@ import { Request, Response } from 'express' export interface IAdminAuthProvider { handleLogin(request: Request, response: Response): Promise + handleLogout(request: Request, response: Response): void isRequestAuthenticated(request: Request): boolean getSessionExpiresAt(request: Request): number | undefined } diff --git a/src/admin/password-admin-auth-provider.ts b/src/admin/password-admin-auth-provider.ts index 3b6f9dfc..9111934a 100644 --- a/src/admin/password-admin-auth-provider.ts +++ b/src/admin/password-admin-auth-provider.ts @@ -5,6 +5,7 @@ import { Settings } from '../@types/settings' import { adminLoginBodySchema } from '../schemas/admin-login-schema' import { verifyAdminPasswordHash, verifyPlaintextPassword } from '../utils/admin-password' import { + buildAdminSessionClearCookieHeader, buildAdminSessionCookieHeader, createAdminSessionToken, getAdminSessionTokenFromRequest, @@ -46,6 +47,16 @@ export class PasswordAdminAuthProvider implements IAdminAuthProvider { } } + public handleLogout(request: Request, response: Response): void { + const currentSettings = this.settings() + + response + .status(200) + .setHeader('content-type', 'application/json') + .setHeader('Set-Cookie', buildAdminSessionClearCookieHeader(request, currentSettings)) + .send({ authenticated: false }) + } + public isRequestAuthenticated(request: Request): boolean { const token = this.getToken(request) return token ? isValidAdminSessionToken(token) : false diff --git a/src/app/app.ts b/src/app/app.ts index 8569493b..cccdc607 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -52,7 +52,7 @@ export class App implements IRunnable { const port = process.env.RELAY_PORT ? Number(process.env.RELAY_PORT) : 8008 const logCentered = (input: string, width: number) => { - const start = (width - input.length) >> 1 + const start = Math.max(0, (width - input.length) >> 1) logger.info(' '.repeat(start), input) } logCentered(`v${packageJson.version}`, width) diff --git a/src/controllers/admin/get-metrics-controller.ts b/src/controllers/admin/get-metrics-controller.ts new file mode 100644 index 00000000..0ff140f1 --- /dev/null +++ b/src/controllers/admin/get-metrics-controller.ts @@ -0,0 +1,86 @@ +import { Request, Response } from 'express' + +import { IController } from '../../@types/controllers' +import { createLogger } from '../../factories/logger-factory' +import { + collectAdminMetricsSnapshot, + createUnavailableAdminMetricsSnapshot, + getAdminMetricsSseIntervalMs, +} from '../../utils/admin-metrics' +import { delayMs } from '../../utils/misc' + +const logger = createLogger('get-admin-metrics-controller') + +const getAdminMetricsSnapshotTimeoutMs = (): number => { + const timeoutCandidate = Number(process.env.ADMIN_METRICS_SNAPSHOT_TIMEOUT_MS) + if (!Number.isFinite(timeoutCandidate) || timeoutCandidate < 1000) { + return 10000 + } + + return timeoutCandidate +} + +export class GetAdminMetricsController implements IController { + public async handleRequest(request: Request, response: Response): Promise { + response.status(200) + response.setHeader('content-type', 'text/event-stream; charset=utf-8') + response.setHeader('cache-control', 'no-cache, no-transform') + response.setHeader('connection', 'keep-alive') + response.flushHeaders() + + let closed = false + let inFlight = false + let timer: ReturnType | undefined + + const cleanup = () => { + closed = true + if (timer) { + clearInterval(timer) + timer = undefined + } + } + + request.on('close', cleanup) + + const sendSnapshot = async () => { + if (closed || response.writableEnded || inFlight) { + return + } + + inFlight = true + + try { + const snapshot = await Promise.race([ + collectAdminMetricsSnapshot(), + delayMs(getAdminMetricsSnapshotTimeoutMs()).then(() => { + throw new Error('admin metrics snapshot timed out') + }), + ]) + if (closed || response.writableEnded) { + return + } + + response.write(`data: ${JSON.stringify(snapshot)}\n\n`) + } catch (error) { + logger.warn('failed to collect admin metrics snapshot: %o', error) + if (!closed && !response.writableEnded) { + const fallbackSnapshot = createUnavailableAdminMetricsSnapshot('failed to collect metrics') + response.write(`data: ${JSON.stringify(fallbackSnapshot)}\n\n`) + } + } finally { + inFlight = false + } + } + + await sendSnapshot() + + if (closed || response.writableEnded) { + return + } + + const intervalMs = getAdminMetricsSseIntervalMs() + timer = setInterval(() => { + void sendSnapshot() + }, intervalMs) + } +} diff --git a/src/controllers/admin/post-logout-controller.ts b/src/controllers/admin/post-logout-controller.ts new file mode 100644 index 00000000..50fc7829 --- /dev/null +++ b/src/controllers/admin/post-logout-controller.ts @@ -0,0 +1,12 @@ +import { Request, Response } from 'express' + +import { IAdminAuthProvider } from '../../@types/admin' +import { IController } from '../../@types/controllers' + +export class PostAdminLogoutController implements IController { + public constructor(private readonly authProvider: IAdminAuthProvider) {} + + public async handleRequest(request: Request, response: Response): Promise { + this.authProvider.handleLogout(request, response) + } +} diff --git a/src/factories/controllers/get-admin-metrics-controller-factory.ts b/src/factories/controllers/get-admin-metrics-controller-factory.ts new file mode 100644 index 00000000..389ce867 --- /dev/null +++ b/src/factories/controllers/get-admin-metrics-controller-factory.ts @@ -0,0 +1,6 @@ +import { GetAdminMetricsController } from '../../controllers/admin/get-metrics-controller' +import { IController } from '../../@types/controllers' + +export const createGetAdminMetricsController = (): IController => { + return new GetAdminMetricsController() +} diff --git a/src/factories/controllers/post-admin-logout-controller-factory.ts b/src/factories/controllers/post-admin-logout-controller-factory.ts new file mode 100644 index 00000000..3b11e0ac --- /dev/null +++ b/src/factories/controllers/post-admin-logout-controller-factory.ts @@ -0,0 +1,7 @@ +import { PostAdminLogoutController } from '../../controllers/admin/post-logout-controller' +import { IController } from '../../@types/controllers' +import { createAdminAuthProvider } from '../admin-auth-provider-factory' + +export const createPostAdminLogoutController = (): IController => { + return new PostAdminLogoutController(createAdminAuthProvider()) +} diff --git a/src/factories/web-app-factory.ts b/src/factories/web-app-factory.ts index 697b262c..88112ab5 100644 --- a/src/factories/web-app-factory.ts +++ b/src/factories/web-app-factory.ts @@ -3,6 +3,7 @@ import { randomBytes } from 'crypto' import { createSettings } from './settings-factory' import router from '../routes' +import { getGrafanaFrameOrigin } from '../utils/admin-grafana' export const createWebApp = (): Express => { const app = express() @@ -15,12 +16,13 @@ export const createWebApp = (): Express => { const relayUrl = new URL(settings.info.relay_url) const webRelayUrl = new URL(relayUrl.toString()) - webRelayUrl.protocol = relayUrl.protocol === 'wss:' ? 'https:' : ':' + webRelayUrl.protocol = relayUrl.protocol === 'wss:' ? 'https:' : 'http:' const directives = { 'img-src': ["'self'", 'data:', 'https://cdn.zebedee.io/an/nostr/'], 'connect-src': ["'self'", settings.info.relay_url as string, webRelayUrl.toString()], 'default-src': ["'self'"], + 'frame-src': ["'self'", getGrafanaFrameOrigin()], 'script-src-attr': [`'nonce-${nonce}'`], 'script-src': [ "'self'", diff --git a/src/handlers/event-message-handler.ts b/src/handlers/event-message-handler.ts index 5ac88b71..cae6541a 100644 --- a/src/handlers/event-message-handler.ts +++ b/src/handlers/event-message-handler.ts @@ -358,7 +358,14 @@ export class EventMessageHandler implements IMessageHandler { continue } - const isRateLimited = await hit({ period, rate, kinds }) + let isRateLimited = false + try { + isRateLimited = await hit({ period, rate, kinds }) + } catch (error) { + // Redis outages should not make the relay reject/abort incoming events. + logger('rate limiter unavailable for %s (%d/%d): %o', event.pubkey, rate, period, error) + continue + } if (isRateLimited) { logger('rate limited %s: %d events / %d ms exceeded', event.pubkey, rate, period) diff --git a/src/handlers/request-handlers/get-admin-dashboard-request-handler.ts b/src/handlers/request-handlers/get-admin-dashboard-request-handler.ts new file mode 100644 index 00000000..34afafe6 --- /dev/null +++ b/src/handlers/request-handlers/get-admin-dashboard-request-handler.ts @@ -0,0 +1,31 @@ +import { NextFunction, Request, Response } from 'express' + +import { createSettings as settings } from '../../factories/settings-factory' +import { getGrafanaBaseUrl, getGrafanaDashboardUid, getGrafanaDashboardUrl, getGrafanaEmbedUrl } from '../../utils/admin-grafana' +import { escapeHtml } from '../../utils/html' +import { getPublicPathPrefix } from '../../utils/http' +import { getTemplate } from '../../utils/template-cache' + +export const getAdminDashboardRequestHandler = (req: Request, res: Response, next: NextFunction) => { + const currentSettings = settings() + const { + info: { name }, + } = currentSettings + + let page: string + try { + page = getTemplate('./resources/admin/dashboard.html') + .replaceAll('{{name}}', escapeHtml(name)) + .replaceAll('{{path_prefix}}', escapeHtml(getPublicPathPrefix(req, currentSettings))) + .replaceAll('{{grafana_embed_url}}', escapeHtml(getGrafanaEmbedUrl())) + .replaceAll('{{grafana_url}}', escapeHtml(getGrafanaDashboardUrl())) + .replaceAll('{{grafana_base_url}}', escapeHtml(getGrafanaBaseUrl())) + .replaceAll('{{grafana_dashboard_uid}}', escapeHtml(getGrafanaDashboardUid())) + .replaceAll('{{nonce}}', res.locals.nonce) + } catch (err) { + next(err) + return + } + + res.status(200).setHeader('content-type', 'text/html; charset=utf8').send(page) +} diff --git a/src/routes/admin/index.ts b/src/routes/admin/index.ts index 0c879219..3e006ca5 100644 --- a/src/routes/admin/index.ts +++ b/src/routes/admin/index.ts @@ -1,14 +1,17 @@ -import { json, Router } from 'express' +import express, { json, Router } from 'express' import { createGetAdminHealthController } from '../../factories/controllers/get-admin-health-controller-factory' +import { createGetAdminMetricsController } from '../../factories/controllers/get-admin-metrics-controller-factory' import { createGetAdminSessionController } from '../../factories/controllers/get-admin-session-controller-factory' import { createPostAdminLoginController } from '../../factories/controllers/post-admin-login-controller-factory' +import { createPostAdminLogoutController } from '../../factories/controllers/post-admin-logout-controller-factory' import { adminAuthMiddleware } from '../../handlers/request-handlers/admin-auth-middleware' import { adminEnabledMiddleware } from '../../handlers/request-handlers/admin-enabled-middleware' import { adminLoginRateLimitMiddleware, adminRateLimitMiddleware, } from '../../handlers/request-handlers/admin-rate-limit-middleware' +import { getAdminDashboardRequestHandler } from '../../handlers/request-handlers/get-admin-dashboard-request-handler' import { rateLimiterMiddleware } from '../../handlers/request-handlers/rate-limiter-middleware' import { withAdminController } from '../../handlers/request-handlers/with-admin-controller-request-handler' @@ -18,8 +21,13 @@ const router: Router = Router() router.use(rateLimiterMiddleware) // codeql[js/missing-rate-limiting] - feature gate only, not authentication router.use(adminEnabledMiddleware) +router.use('/assets', express.static('./resources/admin/assets')) +router.get('/', getAdminDashboardRequestHandler) +router.get('/dashboard', getAdminDashboardRequestHandler) router.post('/login', adminLoginRateLimitMiddleware, json(), withAdminController(createPostAdminLoginController)) +router.post('/logout', adminRateLimitMiddleware, withAdminController(createPostAdminLogoutController)) router.get('/session', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminSessionController)) router.get('/health', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminHealthController)) +router.get('/metrics', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminMetricsController)) export default router diff --git a/src/utils/admin-grafana.ts b/src/utils/admin-grafana.ts new file mode 100644 index 00000000..050fe99a --- /dev/null +++ b/src/utils/admin-grafana.ts @@ -0,0 +1,48 @@ +const DEFAULT_GRAFANA_URL = 'http://127.0.0.1:3000' +const DEFAULT_DASHBOARD_UID = 'nostream-overview' + +export const getGrafanaBaseUrl = (): string => { + const configured = process.env.GRAFANA_URL?.trim() + if (!configured) { + return DEFAULT_GRAFANA_URL + } + + return configured.replace(/\/+$/, '') +} + +export const getGrafanaDashboardUid = (): string => { + const configured = process.env.GRAFANA_DASHBOARD_UID?.trim() + return configured && configured.length > 0 ? configured : DEFAULT_DASHBOARD_UID +} + +export const getGrafanaDashboardUrl = (): string => { + const baseUrl = getGrafanaBaseUrl() + const dashboardUid = getGrafanaDashboardUid() + + return `${baseUrl}/d/${dashboardUid}/nostream-overview?orgId=1&refresh=5s&theme=light` +} + +export const getGrafanaSoloPanelUrl = (panelId: number | string, theme: 'light' | 'dark' = 'light'): string => { + const baseUrl = getGrafanaBaseUrl() + const dashboardUid = getGrafanaDashboardUid() + const params = new URLSearchParams({ + orgId: '1', + panelId: String(panelId), + refresh: '5s', + theme, + }) + + return `${baseUrl}/d-solo/${dashboardUid}?${params.toString()}` +} + +export const getGrafanaEmbedUrl = (): string => { + return `${getGrafanaDashboardUrl()}&kiosk` +} + +export const getGrafanaFrameOrigin = (): string => { + try { + return new URL(getGrafanaBaseUrl()).origin + } catch { + return new URL(DEFAULT_GRAFANA_URL).origin + } +} diff --git a/src/utils/admin-health.ts b/src/utils/admin-health.ts index 903b6c8f..97451e45 100644 --- a/src/utils/admin-health.ts +++ b/src/utils/admin-health.ts @@ -1,5 +1,6 @@ import { getCacheClient } from '../cache/client' import { getMasterDbClient } from '../database/client' +import { delayMs } from './misc' export interface AdminDependencyHealth { ok: boolean @@ -16,9 +17,17 @@ export interface AdminHealthSnapshot { redis: AdminDependencyHealth } +export const getAdminDependencyPingTimeoutMs = (): number => { + const timeoutCandidate = Number(process.env.ADMIN_DEPENDENCY_PING_TIMEOUT_MS) + if (!Number.isFinite(timeoutCandidate) || timeoutCandidate < 100) { + return 3000 + } + + return timeoutCandidate +} + export const collectAdminHealthSnapshot = async (): Promise => { - const database = await pingDatabase() - const redis = await pingRedis() + const [database, redis] = await Promise.all([pingDatabase(), pingRedis()]) return { status: database.ok && redis.ok ? 'ok' : 'degraded', @@ -32,9 +41,20 @@ export const collectAdminHealthSnapshot = async (): Promise } } +const withTimeout = async (operation: Promise, timeoutMs: number): Promise => { + return Promise.race([ + operation, + delayMs(timeoutMs).then(() => { + throw new Error(`dependency ping timed out after ${timeoutMs}ms`) + }), + ]) +} + const pingDatabase = async (): Promise => { + const timeoutMs = getAdminDependencyPingTimeoutMs() + try { - await getMasterDbClient().raw('SELECT 1') + await withTimeout(getMasterDbClient().raw('SELECT 1'), timeoutMs) return { ok: true } } catch { return { ok: false } @@ -42,12 +62,14 @@ const pingDatabase = async (): Promise => { } const pingRedis = async (): Promise => { + const timeoutMs = getAdminDependencyPingTimeoutMs() + try { const client = getCacheClient() if (!client.isOpen) { - await client.connect() + await withTimeout(client.connect(), timeoutMs) } - const pong = await client.ping() + const pong = await withTimeout(client.ping(), timeoutMs) return { ok: pong === 'PONG' } } catch { return { ok: false } diff --git a/src/utils/admin-metrics.ts b/src/utils/admin-metrics.ts new file mode 100644 index 00000000..2ab68ecb --- /dev/null +++ b/src/utils/admin-metrics.ts @@ -0,0 +1,171 @@ +import { AdminDependencyHealth, collectAdminHealthSnapshot } from './admin-health' +import { queryPrometheusInstant } from './prometheus-client' + +export interface AdminMetricsValues { + eventsPerSecond: number + eventsRejectedPerSecond: number + eventsAcceptedTotal: number + eventsRejectedTotal: number + activeConnections: number + cpuLoadPercent: number + memoryUsedMb: number +} + +export interface AdminMetricsSnapshot { + timestamp: number + status: 'ok' | 'degraded' | 'unavailable' + metrics: AdminMetricsValues + health: { + database: AdminDependencyHealth + redis: AdminDependencyHealth + } + prometheus: { + available: boolean + error?: string + } +} + +const PROMQL = { + eventsAcceptedRate: 'sum(rate(nostream_events_accepted_total[1m]))', + eventsRejectedRate: 'sum(rate(nostream_events_rejected_total[1m]))', + eventsAcceptedTotal: 'sum(nostream_events_accepted_total)', + eventsRejectedTotal: 'sum(nostream_events_rejected_total)', + activeConnections: 'sum(nostream_websocket_connections)', + cpuLoadPercent: 'avg(nostream_process_cpu_load_percent)', + memoryUsedMb: 'sum(nostream_process_memory_used_mb)', +} as const + +const toMetricValue = (value: number | undefined): number => { + return value ?? 0 +} + +const emptyMetrics: AdminMetricsValues = { + eventsPerSecond: 0, + eventsRejectedPerSecond: 0, + eventsAcceptedTotal: 0, + eventsRejectedTotal: 0, + activeConnections: 0, + cpuLoadPercent: 0, + memoryUsedMb: 0, +} + +export const createUnavailableAdminMetricsSnapshot = (error: string): AdminMetricsSnapshot => ({ + timestamp: Date.now(), + status: 'unavailable', + metrics: { ...emptyMetrics }, + health: { + database: { ok: false }, + redis: { ok: false }, + }, + prometheus: { + available: false, + error, + }, +}) + +let cachedSnapshot: AdminMetricsSnapshot | undefined +let cachedAt = 0 +let snapshotInFlight: Promise | undefined + +export const resetAdminMetricsSnapshotCache = (): void => { + cachedSnapshot = undefined + cachedAt = 0 + snapshotInFlight = undefined +} + +const collectAdminMetricsSnapshotUncached = async (): Promise => { + const [ + health, + eventsAcceptedRate, + eventsRejectedRate, + eventsAcceptedTotal, + eventsRejectedTotal, + activeConnections, + cpuLoadPercent, + memoryUsedMb, + ] = await Promise.all([ + collectAdminHealthSnapshot(), + queryPrometheusInstant(PROMQL.eventsAcceptedRate), + queryPrometheusInstant(PROMQL.eventsRejectedRate), + queryPrometheusInstant(PROMQL.eventsAcceptedTotal), + queryPrometheusInstant(PROMQL.eventsRejectedTotal), + queryPrometheusInstant(PROMQL.activeConnections), + queryPrometheusInstant(PROMQL.cpuLoadPercent), + queryPrometheusInstant(PROMQL.memoryUsedMb), + ]) + + const prometheusAvailable = + eventsAcceptedRate !== undefined || + eventsRejectedRate !== undefined || + eventsAcceptedTotal !== undefined || + eventsRejectedTotal !== undefined || + activeConnections !== undefined || + cpuLoadPercent !== undefined || + memoryUsedMb !== undefined + + const metricsHealthy = health.database.ok && health.redis.ok + let status: AdminMetricsSnapshot['status'] = 'ok' + + if (!prometheusAvailable) { + status = 'unavailable' + } else if (!metricsHealthy) { + status = 'degraded' + } + + return { + timestamp: Date.now(), + status, + metrics: { + eventsPerSecond: toMetricValue(eventsAcceptedRate), + eventsRejectedPerSecond: toMetricValue(eventsRejectedRate), + eventsAcceptedTotal: toMetricValue(eventsAcceptedTotal), + eventsRejectedTotal: toMetricValue(eventsRejectedTotal), + activeConnections: toMetricValue(activeConnections), + cpuLoadPercent: toMetricValue(cpuLoadPercent), + memoryUsedMb: toMetricValue(memoryUsedMb), + }, + health: { + database: health.database, + redis: health.redis, + }, + prometheus: { + available: prometheusAvailable, + ...(prometheusAvailable ? {} : { error: 'Prometheus query returned no data' }), + }, + } +} + +export const collectAdminMetricsSnapshot = async (): Promise => { + const cacheTtlMs = getAdminMetricsSseIntervalMs() + const now = Date.now() + + if (cachedSnapshot && now - cachedAt < cacheTtlMs) { + return cachedSnapshot + } + + if (snapshotInFlight) { + return snapshotInFlight + } + + snapshotInFlight = collectAdminMetricsSnapshotUncached() + .then((snapshot) => { + cachedSnapshot = snapshot + cachedAt = Date.now() + return snapshot + }) + .finally(() => { + snapshotInFlight = undefined + }) + + return snapshotInFlight +} + +export const getAdminMetricsSseIntervalMs = (): number => { + const intervalCandidate = Number(process.env.ADMIN_METRICS_SSE_INTERVAL_MS || 5000) + + if (!Number.isFinite(intervalCandidate) || intervalCandidate < 1000) { + return 5000 + } + + return intervalCandidate +} diff --git a/src/utils/admin-session.ts b/src/utils/admin-session.ts index 124e08a0..e83ed25c 100644 --- a/src/utils/admin-session.ts +++ b/src/utils/admin-session.ts @@ -30,6 +30,13 @@ export const buildAdminSessionCookieHeader = ( return `admin_session=${token}; Path=${cookiePath}; HttpOnly; SameSite=Strict; Max-Age=${maxAgeSeconds}${secure}` } +export const buildAdminSessionClearCookieHeader = (request: IncomingMessage, settings: Settings): string => { + const cookiePath = joinPathPrefix(getPublicPathPrefix(request, settings), '/admin') + const secure = isSecureRequest(request, settings) ? '; Secure' : '' + + return `admin_session=; Path=${cookiePath}; HttpOnly; SameSite=Strict; Max-Age=0${secure}` +} + export const createAdminSessionToken = (expiresAt: number): string => { const signature = hmacSha256(deriveFromSecret('admin-session'), `${expiresAt}`).toString('hex') return `${expiresAt}.${signature}` diff --git a/src/utils/prometheus-client.ts b/src/utils/prometheus-client.ts new file mode 100644 index 00000000..24f6374c --- /dev/null +++ b/src/utils/prometheus-client.ts @@ -0,0 +1,83 @@ +import axios from 'axios' + +import { createLogger } from '../factories/logger-factory' + +const logger = createLogger('prometheus-client') + +const DEFAULT_PROMETHEUS_URL = 'http://127.0.0.1:9090' + +interface PrometheusInstantQueryResponse { + status: 'success' | 'error' + data?: { + resultType: 'vector' | 'scalar' | string + result: + | Array<{ + value?: [number, string] + }> + | [number, string] + } + error?: string + errorType?: string +} + +export const getPrometheusBaseUrl = (): string => { + const configured = process.env.PROMETHEUS_URL?.trim() + if (!configured) { + return DEFAULT_PROMETHEUS_URL + } + + return configured.replace(/\/+$/, '') +} + +export const parsePrometheusInstantQueryScalar = (response: PrometheusInstantQueryResponse): number | undefined => { + if (response.status !== 'success') { + return undefined + } + + const data = response.data + let rawValue: string | undefined + + if (data?.resultType === 'scalar' && Array.isArray(data.result) && data.result.length >= 2) { + rawValue = String(data.result[1]) + } else if (Array.isArray(data?.result)) { + const vectorResult = data.result as Array<{ value?: [number, string] }> + rawValue = vectorResult[0]?.value?.[1] + } + + if (rawValue === undefined) { + return undefined + } + + const parsed = Number(rawValue) + return Number.isFinite(parsed) ? parsed : undefined +} + +export const queryPrometheusInstant = async (query: string): Promise => { + const baseUrl = getPrometheusBaseUrl() + + try { + const timeoutCandidate = Number(process.env.PROMETHEUS_QUERY_TIMEOUT_MS) + const timeoutMs = Number.isFinite(timeoutCandidate) && timeoutCandidate > 0 ? timeoutCandidate : 5000 + + const response = await axios.get(`${baseUrl}/api/v1/query`, { + params: { query }, + timeout: timeoutMs, + validateStatus: () => true, + }) + + if (response.status !== 200) { + logger.warn('prometheus query failed with HTTP %d: %s', response.status, query) + return undefined + } + + if (response.data.status !== 'success') { + logger.warn('prometheus query error for %s: %s', query, response.data.error ?? response.data.errorType ?? 'unknown') + return undefined + } + + return parsePrometheusInstantQueryScalar(response.data) + } catch (error) { + logger.warn('prometheus query request failed for %s: %o', query, error) + return undefined + } +} diff --git a/test/unit/handlers/event-message-handler.spec.ts b/test/unit/handlers/event-message-handler.spec.ts index 14c894da..be2631c3 100644 --- a/test/unit/handlers/event-message-handler.spec.ts +++ b/test/unit/handlers/event-message-handler.spec.ts @@ -1022,6 +1022,21 @@ describe('EventMessageHandler', () => { ) expect(actualResult).to.be.true }) + + it('fails open when the rate limiter backend is unavailable', async () => { + eventLimits.rateLimits = [ + { + period: 60000, + rate: 1, + }, + ] + rateLimiterHitStub.rejects(new Error('redis unavailable')) + + const actualResult = await (handler as any).isRateLimited(event) + + expect(actualResult).to.be.false + expect(rateLimiterHitStub).to.have.been.calledOnce + }) }) describe('isUserAdmitted', () => { diff --git a/test/unit/routes/admin.spec.ts b/test/unit/routes/admin.spec.ts index c7c852ce..f4933c15 100644 --- a/test/unit/routes/admin.spec.ts +++ b/test/unit/routes/admin.spec.ts @@ -4,6 +4,7 @@ import express from 'express' import Sinon from 'sinon' import * as getAdminHealthControllerFactory from '../../../src/factories/controllers/get-admin-health-controller-factory' +import * as getAdminMetricsControllerFactory from '../../../src/factories/controllers/get-admin-metrics-controller-factory' import { hashAdminPassword } from '../../../src/utils/admin-password' import * as adminRateLimitMiddleware from '../../../src/handlers/request-handlers/admin-rate-limit-middleware' import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware' @@ -13,6 +14,7 @@ describe('admin router', () => { const originalSecret = process.env.SECRET const originalAdminPassword = process.env.ADMIN_PASSWORD let createGetAdminHealthControllerStub: Sinon.SinonStub + let createGetAdminMetricsControllerStub: Sinon.SinonStub let createSettingsStub: Sinon.SinonStub let rateLimiterMiddlewareStub: Sinon.SinonStub let adminRateLimitMiddlewareStub: Sinon.SinonStub @@ -48,6 +50,18 @@ describe('admin router', () => { ) }, } as any) + createGetAdminMetricsControllerStub = Sinon.stub( + getAdminMetricsControllerFactory, + 'createGetAdminMetricsController', + ).returns({ + handleRequest: async (_request: any, response: any) => { + response + .status(200) + .setHeader('content-type', 'text/event-stream; charset=utf-8') + .write('data: {"timestamp":1,"status":"ok"}\n\n') + response.end() + }, + } as any) createSettingsStub = Sinon.stub(settingsFactory, 'createSettings').returns(settings as any) const passthrough = async (_request: any, _response: any, next: any) => { next() @@ -61,6 +75,10 @@ describe('admin router', () => { ) const router = loadAdminRouter() const app = express() + app.use((request, response, next) => { + response.locals.nonce = 'test-admin-nonce' + next() + }) app.use('/admin', router) server = await new Promise((resolve) => { @@ -72,6 +90,7 @@ describe('admin router', () => { const stopServer = async () => { createGetAdminHealthControllerStub?.restore() + createGetAdminMetricsControllerStub?.restore() createSettingsStub?.restore() rateLimiterMiddlewareStub?.restore() adminRateLimitMiddlewareStub?.restore() @@ -120,10 +139,26 @@ describe('admin router', () => { it('returns 404 when admin is disabled', async () => { const baseUrl = await startServer({ admin: { enabled: false } }) - const response = await axios.get(`${baseUrl}/health`, { validateStatus: () => true }) + const healthResponse = await axios.get(`${baseUrl}/health`, { validateStatus: () => true }) + const dashboardResponse = await axios.get(`${baseUrl}/`, { validateStatus: () => true }) - expect(response.status).to.equal(404) - expect(response.data).to.equal('Not Found') + expect(healthResponse.status).to.equal(404) + expect(dashboardResponse.status).to.equal(404) + expect(healthResponse.data).to.equal('Not Found') + expect(rateLimiterMiddlewareStub.calledTwice).to.be.true + }) + + it('serves the dashboard shell when admin is enabled', async () => { + const baseUrl = await startServer({ admin: { enabled: true }, info: { name: 'Test Relay' } }) + + const response = await axios.get(`${baseUrl}/`, { validateStatus: () => true }) + + expect(response.status).to.equal(200) + expect(response.headers['content-type']).to.include('text/html') + expect(response.data).to.include('Test Relay') + expect(response.data).to.include('admin/assets/dashboard.js') + expect(response.data).to.include('grafana-frame') + expect(response.data).to.include('nostream-overview') expect(rateLimiterMiddlewareStub.calledOnce).to.be.true }) @@ -132,10 +167,12 @@ describe('admin router', () => { const sessionResponse = await axios.get(`${baseUrl}/session`, { validateStatus: () => true }) const healthResponse = await axios.get(`${baseUrl}/health`, { validateStatus: () => true }) + const metricsResponse = await axios.get(`${baseUrl}/metrics`, { validateStatus: () => true }) expect(sessionResponse.status).to.equal(401) expect(healthResponse.status).to.equal(401) - expect(rateLimiterMiddlewareStub.calledTwice).to.be.true + expect(metricsResponse.status).to.equal(401) + expect(rateLimiterMiddlewareStub.callCount).to.equal(3) }) it('rejects invalid login credentials', async () => { @@ -229,9 +266,18 @@ describe('admin router', () => { expect(healthResponse.status).to.equal(200) expect(healthResponse.data).to.include.keys('status', 'uptimeSeconds', 'worker', 'database', 'redis') - expect(rateLimiterMiddlewareStub.callCount).to.equal(3) + const metricsResponse = await axios.get(`${baseUrl}/metrics`, { + headers: { cookie }, + validateStatus: () => true, + responseType: 'text', + }) + expect(metricsResponse.status).to.equal(200) + expect(metricsResponse.headers['content-type']).to.include('text/event-stream') + expect(metricsResponse.data).to.include('"timestamp":1') + + expect(rateLimiterMiddlewareStub.callCount).to.equal(4) expect(adminLoginRateLimitMiddlewareStub.calledOnce).to.be.true - expect(adminRateLimitMiddlewareStub.calledTwice).to.be.true + expect(adminRateLimitMiddlewareStub.callCount).to.equal(3) }) it('authenticates with passwordHash from settings', async () => { diff --git a/test/unit/utils/admin-grafana.spec.ts b/test/unit/utils/admin-grafana.spec.ts new file mode 100644 index 00000000..1318c57b --- /dev/null +++ b/test/unit/utils/admin-grafana.spec.ts @@ -0,0 +1,60 @@ +import { expect } from 'chai' + +import { + getGrafanaBaseUrl, + getGrafanaDashboardUid, + getGrafanaDashboardUrl, + getGrafanaEmbedUrl, + getGrafanaFrameOrigin, + getGrafanaSoloPanelUrl, +} from '../../../src/utils/admin-grafana' + +describe('admin-grafana', () => { + const originalGrafanaUrl = process.env.GRAFANA_URL + const originalDashboardUid = process.env.GRAFANA_DASHBOARD_UID + + afterEach(() => { + if (originalGrafanaUrl === undefined) { + delete process.env.GRAFANA_URL + } else { + process.env.GRAFANA_URL = originalGrafanaUrl + } + + if (originalDashboardUid === undefined) { + delete process.env.GRAFANA_DASHBOARD_UID + } else { + process.env.GRAFANA_DASHBOARD_UID = originalDashboardUid + } + }) + + it('uses defaults when env vars are unset', () => { + delete process.env.GRAFANA_URL + delete process.env.GRAFANA_DASHBOARD_UID + + expect(getGrafanaBaseUrl()).to.equal('http://127.0.0.1:3000') + expect(getGrafanaDashboardUid()).to.equal('nostream-overview') + expect(getGrafanaFrameOrigin()).to.equal('http://127.0.0.1:3000') + }) + + it('normalizes configured grafana url', () => { + process.env.GRAFANA_URL = 'http://grafana.example.com/' + + expect(getGrafanaBaseUrl()).to.equal('http://grafana.example.com') + expect(getGrafanaFrameOrigin()).to.equal('http://grafana.example.com') + }) + + it('builds dashboard and embed urls', () => { + process.env.GRAFANA_URL = 'http://127.0.0.1:3000' + process.env.GRAFANA_DASHBOARD_UID = 'custom-dashboard' + + expect(getGrafanaDashboardUrl()).to.equal( + 'http://127.0.0.1:3000/d/custom-dashboard/nostream-overview?orgId=1&refresh=5s&theme=light', + ) + expect(getGrafanaEmbedUrl()).to.equal( + 'http://127.0.0.1:3000/d/custom-dashboard/nostream-overview?orgId=1&refresh=5s&theme=light&kiosk', + ) + expect(getGrafanaSoloPanelUrl(7, 'dark')).to.equal( + 'http://127.0.0.1:3000/d-solo/custom-dashboard?orgId=1&panelId=7&refresh=5s&theme=dark', + ) + }) +}) diff --git a/test/unit/utils/admin-metrics.spec.ts b/test/unit/utils/admin-metrics.spec.ts new file mode 100644 index 00000000..a2ae72ee --- /dev/null +++ b/test/unit/utils/admin-metrics.spec.ts @@ -0,0 +1,129 @@ +import chai from 'chai' +import Sinon from 'sinon' +import sinonChai from 'sinon-chai' + +chai.use(sinonChai) +const { expect } = chai + +import * as adminHealth from '../../../src/utils/admin-health' +import * as adminMetrics from '../../../src/utils/admin-metrics' +import * as prometheusClient from '../../../src/utils/prometheus-client' + +describe('admin-metrics', () => { + let sandbox: Sinon.SinonSandbox + let queryPrometheusInstantStub: Sinon.SinonStub + let collectAdminHealthSnapshotStub: Sinon.SinonStub + + beforeEach(() => { + adminMetrics.resetAdminMetricsSnapshotCache() + sandbox = Sinon.createSandbox() + queryPrometheusInstantStub = sandbox.stub(prometheusClient, 'queryPrometheusInstant') + collectAdminHealthSnapshotStub = sandbox.stub(adminHealth, 'collectAdminHealthSnapshot').resolves({ + status: 'ok', + uptimeSeconds: 10, + worker: { type: 'worker', index: '0' }, + database: { ok: true }, + redis: { ok: true }, + }) + }) + + afterEach(() => { + adminMetrics.resetAdminMetricsSnapshotCache() + sandbox.restore() + }) + + it('builds a snapshot from prometheus queries and dependency health', async () => { + queryPrometheusInstantStub.onCall(0).resolves(1.5) + queryPrometheusInstantStub.onCall(1).resolves(0.25) + queryPrometheusInstantStub.onCall(2).resolves(100) + queryPrometheusInstantStub.onCall(3).resolves(4) + queryPrometheusInstantStub.onCall(4).resolves(12) + queryPrometheusInstantStub.onCall(5).resolves(33.3) + queryPrometheusInstantStub.onCall(6).resolves(256) + + const snapshot = await adminMetrics.collectAdminMetricsSnapshot() + + expect(snapshot.status).to.equal('ok') + expect(snapshot.metrics).to.deep.equal({ + eventsPerSecond: 1.5, + eventsRejectedPerSecond: 0.25, + eventsAcceptedTotal: 100, + eventsRejectedTotal: 4, + activeConnections: 12, + cpuLoadPercent: 33.3, + memoryUsedMb: 256, + }) + expect(snapshot.health.database.ok).to.equal(true) + expect(snapshot.prometheus.available).to.equal(true) + expect(collectAdminHealthSnapshotStub).to.have.been.calledOnce + expect(queryPrometheusInstantStub.callCount).to.equal(7) + }) + + it('builds an unavailable fallback snapshot for SSE error handling', () => { + const snapshot = adminMetrics.createUnavailableAdminMetricsSnapshot('failed to collect metrics') + + expect(snapshot.status).to.equal('unavailable') + expect(snapshot.prometheus.available).to.equal(false) + expect(snapshot.prometheus.error).to.equal('failed to collect metrics') + expect(snapshot.metrics.eventsPerSecond).to.equal(0) + }) + + it('marks snapshot unavailable when prometheus returns no data', async () => { + queryPrometheusInstantStub.resolves(undefined) + + const snapshot = await adminMetrics.collectAdminMetricsSnapshot() + + expect(snapshot.status).to.equal('unavailable') + expect(snapshot.prometheus.available).to.equal(false) + expect(snapshot.metrics.eventsPerSecond).to.equal(0) + }) + + it('marks snapshot degraded when dependency health fails', async () => { + collectAdminHealthSnapshotStub.resolves({ + status: 'degraded', + uptimeSeconds: 10, + worker: { type: 'worker' }, + database: { ok: false }, + redis: { ok: true }, + }) + queryPrometheusInstantStub.onCall(0).resolves(1) + + const snapshot = await adminMetrics.collectAdminMetricsSnapshot() + + expect(snapshot.status).to.equal('degraded') + expect(snapshot.health.database.ok).to.equal(false) + }) + + it('reuses cached snapshots within the SSE interval', async () => { + queryPrometheusInstantStub.resolves(1) + + await adminMetrics.collectAdminMetricsSnapshot() + await adminMetrics.collectAdminMetricsSnapshot() + + expect(queryPrometheusInstantStub.callCount).to.equal(7) + }) + + describe('getAdminMetricsSseIntervalMs', () => { + const originalInterval = process.env.ADMIN_METRICS_SSE_INTERVAL_MS + + afterEach(() => { + if (originalInterval === undefined) { + delete process.env.ADMIN_METRICS_SSE_INTERVAL_MS + } else { + process.env.ADMIN_METRICS_SSE_INTERVAL_MS = originalInterval + } + }) + + it('defaults to 5000ms', () => { + delete process.env.ADMIN_METRICS_SSE_INTERVAL_MS + + expect(adminMetrics.getAdminMetricsSseIntervalMs()).to.equal(5000) + }) + + it('falls back when configured interval is invalid', () => { + process.env.ADMIN_METRICS_SSE_INTERVAL_MS = 'abc' + + expect(adminMetrics.getAdminMetricsSseIntervalMs()).to.equal(5000) + }) + }) +}) diff --git a/test/unit/utils/prometheus-client.spec.ts b/test/unit/utils/prometheus-client.spec.ts new file mode 100644 index 00000000..c6c8a4d7 --- /dev/null +++ b/test/unit/utils/prometheus-client.spec.ts @@ -0,0 +1,110 @@ +import axios from 'axios' +import chai from 'chai' +import Sinon from 'sinon' +import sinonChai from 'sinon-chai' + +chai.use(sinonChai) +const { expect } = chai + +import * as prometheusClient from '../../../src/utils/prometheus-client' + +describe('prometheus-client', () => { + let sandbox: Sinon.SinonSandbox + let axiosGetStub: Sinon.SinonStub + const originalPrometheusUrl = process.env.PROMETHEUS_URL + + beforeEach(() => { + sandbox = Sinon.createSandbox() + axiosGetStub = sandbox.stub(axios, 'get') + delete process.env.PROMETHEUS_URL + }) + + afterEach(() => { + sandbox.restore() + if (originalPrometheusUrl === undefined) { + delete process.env.PROMETHEUS_URL + } else { + process.env.PROMETHEUS_URL = originalPrometheusUrl + } + }) + + describe('getPrometheusBaseUrl', () => { + it('defaults to localhost prometheus', () => { + expect(prometheusClient.getPrometheusBaseUrl()).to.equal('http://127.0.0.1:9090') + }) + + it('trims trailing slashes from configured URL', () => { + process.env.PROMETHEUS_URL = 'http://prometheus:9090/' + + expect(prometheusClient.getPrometheusBaseUrl()).to.equal('http://prometheus:9090') + }) + }) + + describe('parsePrometheusInstantQueryScalar', () => { + it('returns parsed scalar from successful vector response', () => { + const value = prometheusClient.parsePrometheusInstantQueryScalar({ + status: 'success', + data: { + resultType: 'vector', + result: [{ value: [1710000000, '12.5'] }], + }, + }) + + expect(value).to.equal(12.5) + }) + + it('returns parsed scalar from successful scalar response', () => { + const value = prometheusClient.parsePrometheusInstantQueryScalar({ + status: 'success', + data: { + resultType: 'scalar', + result: [1710000000, '9'], + }, + }) + + expect(value).to.equal(9) + }) + + it('returns undefined for unsuccessful responses', () => { + expect( + prometheusClient.parsePrometheusInstantQueryScalar({ + status: 'error', + error: 'bad_data', + }), + ).to.be.undefined + }) + }) + + describe('queryPrometheusInstant', () => { + it('queries prometheus instant API and returns scalar', async () => { + axiosGetStub.resolves({ + status: 200, + data: { + status: 'success', + data: { + resultType: 'vector', + result: [{ value: [1710000000, '3'] }], + }, + }, + }) + + const value = await prometheusClient.queryPrometheusInstant('sum(up)') + + expect(value).to.equal(3) + expect(axiosGetStub).to.have.been.calledOnceWith( + 'http://127.0.0.1:9090/api/v1/query', + Sinon.match({ + params: { query: 'sum(up)' }, + }), + ) + }) + + it('returns undefined when prometheus responds with non-200', async () => { + axiosGetStub.resolves({ status: 503, data: {} }) + + const value = await prometheusClient.queryPrometheusInstant('sum(up)') + + expect(value).to.be.undefined + }) + }) +})