diff --git a/.changeset/admin-metrics-sse-endpoint.md b/.changeset/admin-metrics-sse-endpoint.md new file mode 100644 index 00000000..2431746d --- /dev/null +++ b/.changeset/admin-metrics-sse-endpoint.md @@ -0,0 +1,5 @@ +--- +"nostream": minor +--- + +feat: add Prometheus-backed admin metrics SSE endpoint diff --git a/.env.example b/.env.example index 66c82e1d..060b4b69 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,11 @@ WORKER_COUNT=2 # Defaults to CPU count. Use 1 or 2 for local testing. # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 # OTEL_METRIC_EXPORT_INTERVAL_MS=10000 +# --- OBSERVABILITY (Prometheus for Admin Metrics) --- +# PROMETHEUS_URL=http://127.0.0.1:9090 +# PROMETHEUS_QUERY_TIMEOUT_MS=5000 +# ADMIN_METRICS_SSE_INTERVAL_MS=5000 + # --- RELAY PRIVATE KEY (Optional) --- # RELAY_PRIVATE_KEY=your_hex_private_key diff --git a/docker-compose.yml b/docker-compose.yml index 294cc173..ca0c1177 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,8 @@ services: OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-nostream} OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} OTEL_METRIC_EXPORT_INTERVAL_MS: ${OTEL_METRIC_EXPORT_INTERVAL_MS:-10000} + PROMETHEUS_URL: ${PROMETHEUS_URL:-http://prometheus:9090} + ADMIN_METRICS_SSE_INTERVAL_MS: ${ADMIN_METRICS_SSE_INTERVAL_MS:-5000} # Enable DEBUG for troubleshooting. Examples: # DEBUG: "primary:*" # DEBUG: "worker:*" diff --git a/src/controllers/admin/get-metrics-controller.ts b/src/controllers/admin/get-metrics-controller.ts new file mode 100644 index 00000000..fadeb138 --- /dev/null +++ b/src/controllers/admin/get-metrics-controller.ts @@ -0,0 +1,66 @@ +import { Request, Response } from 'express' + +import { IController } from '../../@types/controllers' +import { createLogger } from '../../factories/logger-factory' +import { collectAdminMetricsSnapshot, getAdminMetricsSseIntervalMs } from '../../utils/admin-metrics' + +const logger = createLogger('get-admin-metrics-controller') + +export class GetAdminMetricsController implements IController { + public async handleRequest(request: Request, response: Response): Promise { + response.status(200) + response.setHeader('content-type', 'text/event-stream; charset=utf-8') + response.setHeader('cache-control', 'no-cache, no-transform') + response.setHeader('connection', 'keep-alive') + response.flushHeaders() + + let closed = false + let inFlight = false + let timer: ReturnType | undefined + + const cleanup = () => { + closed = true + if (timer) { + clearInterval(timer) + timer = undefined + } + } + + request.on('close', cleanup) + + const sendSnapshot = async () => { + if (closed || response.writableEnded || inFlight) { + return + } + + inFlight = true + + try { + const snapshot = await collectAdminMetricsSnapshot() + if (closed || response.writableEnded) { + return + } + + response.write(`data: ${JSON.stringify(snapshot)}\n\n`) + } catch (error) { + logger.warn('failed to collect admin metrics snapshot: %o', error) + if (!closed && !response.writableEnded) { + response.write(`event: error\ndata: ${JSON.stringify({ error: 'failed to collect metrics' })}\n\n`) + } + } finally { + inFlight = false + } + } + + await sendSnapshot() + + if (closed || response.writableEnded) { + return + } + + const intervalMs = getAdminMetricsSseIntervalMs() + timer = setInterval(() => { + void sendSnapshot() + }, intervalMs) + } +} diff --git a/src/factories/controllers/get-admin-metrics-controller-factory.ts b/src/factories/controllers/get-admin-metrics-controller-factory.ts new file mode 100644 index 00000000..389ce867 --- /dev/null +++ b/src/factories/controllers/get-admin-metrics-controller-factory.ts @@ -0,0 +1,6 @@ +import { GetAdminMetricsController } from '../../controllers/admin/get-metrics-controller' +import { IController } from '../../@types/controllers' + +export const createGetAdminMetricsController = (): IController => { + return new GetAdminMetricsController() +} diff --git a/src/routes/admin/index.ts b/src/routes/admin/index.ts index 0c879219..8349906a 100644 --- a/src/routes/admin/index.ts +++ b/src/routes/admin/index.ts @@ -1,6 +1,7 @@ import { json, Router } from 'express' import { createGetAdminHealthController } from '../../factories/controllers/get-admin-health-controller-factory' +import { createGetAdminMetricsController } from '../../factories/controllers/get-admin-metrics-controller-factory' import { createGetAdminSessionController } from '../../factories/controllers/get-admin-session-controller-factory' import { createPostAdminLoginController } from '../../factories/controllers/post-admin-login-controller-factory' import { adminAuthMiddleware } from '../../handlers/request-handlers/admin-auth-middleware' @@ -21,5 +22,6 @@ router.use(adminEnabledMiddleware) router.post('/login', adminLoginRateLimitMiddleware, json(), withAdminController(createPostAdminLoginController)) router.get('/session', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminSessionController)) router.get('/health', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminHealthController)) +router.get('/metrics', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminMetricsController)) export default router diff --git a/src/utils/admin-metrics.ts b/src/utils/admin-metrics.ts new file mode 100644 index 00000000..8c5f9212 --- /dev/null +++ b/src/utils/admin-metrics.ts @@ -0,0 +1,147 @@ +import { AdminDependencyHealth, collectAdminHealthSnapshot } from './admin-health' +import { queryPrometheusInstant } from './prometheus-client' + +export interface AdminMetricsValues { + eventsPerSecond: number + eventsRejectedPerSecond: number + eventsAcceptedTotal: number + eventsRejectedTotal: number + activeConnections: number + cpuLoadPercent: number + memoryUsedMb: number +} + +export interface AdminMetricsSnapshot { + timestamp: number + status: 'ok' | 'degraded' | 'unavailable' + metrics: AdminMetricsValues + health: { + database: AdminDependencyHealth + redis: AdminDependencyHealth + } + prometheus: { + available: boolean + error?: string + } +} + +const PROMQL = { + eventsAcceptedRate: 'sum(rate(nostream_events_accepted_total[1m]))', + eventsRejectedRate: 'sum(rate(nostream_events_rejected_total[1m]))', + eventsAcceptedTotal: 'sum(nostream_events_accepted_total)', + eventsRejectedTotal: 'sum(nostream_events_rejected_total)', + activeConnections: 'sum(nostream_websocket_connections)', + cpuLoadPercent: 'avg(nostream_process_cpu_load_percent)', + memoryUsedMb: 'sum(nostream_process_memory_used_mb)', +} as const + +const toMetricValue = (value: number | undefined): number => { + return value ?? 0 +} + +let cachedSnapshot: AdminMetricsSnapshot | undefined +let cachedAt = 0 +let snapshotInFlight: Promise | undefined + +export const resetAdminMetricsSnapshotCache = (): void => { + cachedSnapshot = undefined + cachedAt = 0 + snapshotInFlight = undefined +} + +const collectAdminMetricsSnapshotUncached = async (): Promise => { + const [ + health, + eventsAcceptedRate, + eventsRejectedRate, + eventsAcceptedTotal, + eventsRejectedTotal, + activeConnections, + cpuLoadPercent, + memoryUsedMb, + ] = await Promise.all([ + collectAdminHealthSnapshot(), + queryPrometheusInstant(PROMQL.eventsAcceptedRate), + queryPrometheusInstant(PROMQL.eventsRejectedRate), + queryPrometheusInstant(PROMQL.eventsAcceptedTotal), + queryPrometheusInstant(PROMQL.eventsRejectedTotal), + queryPrometheusInstant(PROMQL.activeConnections), + queryPrometheusInstant(PROMQL.cpuLoadPercent), + queryPrometheusInstant(PROMQL.memoryUsedMb), + ]) + + const prometheusAvailable = + eventsAcceptedRate !== undefined || + eventsRejectedRate !== undefined || + eventsAcceptedTotal !== undefined || + eventsRejectedTotal !== undefined || + activeConnections !== undefined || + cpuLoadPercent !== undefined || + memoryUsedMb !== undefined + + const metricsHealthy = health.database.ok && health.redis.ok + let status: AdminMetricsSnapshot['status'] = 'ok' + + if (!prometheusAvailable) { + status = 'unavailable' + } else if (!metricsHealthy) { + status = 'degraded' + } + + return { + timestamp: Date.now(), + status, + metrics: { + eventsPerSecond: toMetricValue(eventsAcceptedRate), + eventsRejectedPerSecond: toMetricValue(eventsRejectedRate), + eventsAcceptedTotal: toMetricValue(eventsAcceptedTotal), + eventsRejectedTotal: toMetricValue(eventsRejectedTotal), + activeConnections: toMetricValue(activeConnections), + cpuLoadPercent: toMetricValue(cpuLoadPercent), + memoryUsedMb: toMetricValue(memoryUsedMb), + }, + health: { + database: health.database, + redis: health.redis, + }, + prometheus: { + available: prometheusAvailable, + ...(prometheusAvailable ? {} : { error: 'Prometheus query returned no data' }), + }, + } +} + +export const collectAdminMetricsSnapshot = async (): Promise => { + const cacheTtlMs = getAdminMetricsSseIntervalMs() + const now = Date.now() + + if (cachedSnapshot && now - cachedAt < cacheTtlMs) { + return cachedSnapshot + } + + if (snapshotInFlight) { + return snapshotInFlight + } + + snapshotInFlight = collectAdminMetricsSnapshotUncached() + .then((snapshot) => { + cachedSnapshot = snapshot + cachedAt = Date.now() + return snapshot + }) + .finally(() => { + snapshotInFlight = undefined + }) + + return snapshotInFlight +} + +export const getAdminMetricsSseIntervalMs = (): number => { + const intervalCandidate = Number(process.env.ADMIN_METRICS_SSE_INTERVAL_MS || 5000) + + if (!Number.isFinite(intervalCandidate) || intervalCandidate < 1000) { + return 5000 + } + + return intervalCandidate +} diff --git a/src/utils/prometheus-client.ts b/src/utils/prometheus-client.ts new file mode 100644 index 00000000..24f6374c --- /dev/null +++ b/src/utils/prometheus-client.ts @@ -0,0 +1,83 @@ +import axios from 'axios' + +import { createLogger } from '../factories/logger-factory' + +const logger = createLogger('prometheus-client') + +const DEFAULT_PROMETHEUS_URL = 'http://127.0.0.1:9090' + +interface PrometheusInstantQueryResponse { + status: 'success' | 'error' + data?: { + resultType: 'vector' | 'scalar' | string + result: + | Array<{ + value?: [number, string] + }> + | [number, string] + } + error?: string + errorType?: string +} + +export const getPrometheusBaseUrl = (): string => { + const configured = process.env.PROMETHEUS_URL?.trim() + if (!configured) { + return DEFAULT_PROMETHEUS_URL + } + + return configured.replace(/\/+$/, '') +} + +export const parsePrometheusInstantQueryScalar = (response: PrometheusInstantQueryResponse): number | undefined => { + if (response.status !== 'success') { + return undefined + } + + const data = response.data + let rawValue: string | undefined + + if (data?.resultType === 'scalar' && Array.isArray(data.result) && data.result.length >= 2) { + rawValue = String(data.result[1]) + } else if (Array.isArray(data?.result)) { + const vectorResult = data.result as Array<{ value?: [number, string] }> + rawValue = vectorResult[0]?.value?.[1] + } + + if (rawValue === undefined) { + return undefined + } + + const parsed = Number(rawValue) + return Number.isFinite(parsed) ? parsed : undefined +} + +export const queryPrometheusInstant = async (query: string): Promise => { + const baseUrl = getPrometheusBaseUrl() + + try { + const timeoutCandidate = Number(process.env.PROMETHEUS_QUERY_TIMEOUT_MS) + const timeoutMs = Number.isFinite(timeoutCandidate) && timeoutCandidate > 0 ? timeoutCandidate : 5000 + + const response = await axios.get(`${baseUrl}/api/v1/query`, { + params: { query }, + timeout: timeoutMs, + validateStatus: () => true, + }) + + if (response.status !== 200) { + logger.warn('prometheus query failed with HTTP %d: %s', response.status, query) + return undefined + } + + if (response.data.status !== 'success') { + logger.warn('prometheus query error for %s: %s', query, response.data.error ?? response.data.errorType ?? 'unknown') + return undefined + } + + return parsePrometheusInstantQueryScalar(response.data) + } catch (error) { + logger.warn('prometheus query request failed for %s: %o', query, error) + return undefined + } +} diff --git a/test/unit/routes/admin.spec.ts b/test/unit/routes/admin.spec.ts index c7c852ce..696d8357 100644 --- a/test/unit/routes/admin.spec.ts +++ b/test/unit/routes/admin.spec.ts @@ -4,6 +4,7 @@ import express from 'express' import Sinon from 'sinon' import * as getAdminHealthControllerFactory from '../../../src/factories/controllers/get-admin-health-controller-factory' +import * as getAdminMetricsControllerFactory from '../../../src/factories/controllers/get-admin-metrics-controller-factory' import { hashAdminPassword } from '../../../src/utils/admin-password' import * as adminRateLimitMiddleware from '../../../src/handlers/request-handlers/admin-rate-limit-middleware' import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware' @@ -13,6 +14,7 @@ describe('admin router', () => { const originalSecret = process.env.SECRET const originalAdminPassword = process.env.ADMIN_PASSWORD let createGetAdminHealthControllerStub: Sinon.SinonStub + let createGetAdminMetricsControllerStub: Sinon.SinonStub let createSettingsStub: Sinon.SinonStub let rateLimiterMiddlewareStub: Sinon.SinonStub let adminRateLimitMiddlewareStub: Sinon.SinonStub @@ -48,6 +50,18 @@ describe('admin router', () => { ) }, } as any) + createGetAdminMetricsControllerStub = Sinon.stub( + getAdminMetricsControllerFactory, + 'createGetAdminMetricsController', + ).returns({ + handleRequest: async (_request: any, response: any) => { + response + .status(200) + .setHeader('content-type', 'text/event-stream; charset=utf-8') + .write('data: {"timestamp":1,"status":"ok"}\n\n') + response.end() + }, + } as any) createSettingsStub = Sinon.stub(settingsFactory, 'createSettings').returns(settings as any) const passthrough = async (_request: any, _response: any, next: any) => { next() @@ -72,6 +86,7 @@ describe('admin router', () => { const stopServer = async () => { createGetAdminHealthControllerStub?.restore() + createGetAdminMetricsControllerStub?.restore() createSettingsStub?.restore() rateLimiterMiddlewareStub?.restore() adminRateLimitMiddlewareStub?.restore() @@ -132,10 +147,12 @@ describe('admin router', () => { const sessionResponse = await axios.get(`${baseUrl}/session`, { validateStatus: () => true }) const healthResponse = await axios.get(`${baseUrl}/health`, { validateStatus: () => true }) + const metricsResponse = await axios.get(`${baseUrl}/metrics`, { validateStatus: () => true }) expect(sessionResponse.status).to.equal(401) expect(healthResponse.status).to.equal(401) - expect(rateLimiterMiddlewareStub.calledTwice).to.be.true + expect(metricsResponse.status).to.equal(401) + expect(rateLimiterMiddlewareStub.callCount).to.equal(3) }) it('rejects invalid login credentials', async () => { @@ -229,9 +246,18 @@ describe('admin router', () => { expect(healthResponse.status).to.equal(200) expect(healthResponse.data).to.include.keys('status', 'uptimeSeconds', 'worker', 'database', 'redis') - expect(rateLimiterMiddlewareStub.callCount).to.equal(3) + const metricsResponse = await axios.get(`${baseUrl}/metrics`, { + headers: { cookie }, + validateStatus: () => true, + responseType: 'text', + }) + expect(metricsResponse.status).to.equal(200) + expect(metricsResponse.headers['content-type']).to.include('text/event-stream') + expect(metricsResponse.data).to.include('"timestamp":1') + + expect(rateLimiterMiddlewareStub.callCount).to.equal(4) expect(adminLoginRateLimitMiddlewareStub.calledOnce).to.be.true - expect(adminRateLimitMiddlewareStub.calledTwice).to.be.true + expect(adminRateLimitMiddlewareStub.callCount).to.equal(3) }) it('authenticates with passwordHash from settings', async () => { diff --git a/test/unit/utils/admin-metrics.spec.ts b/test/unit/utils/admin-metrics.spec.ts new file mode 100644 index 00000000..b3bca909 --- /dev/null +++ b/test/unit/utils/admin-metrics.spec.ts @@ -0,0 +1,120 @@ +import chai from 'chai' +import Sinon from 'sinon' +import sinonChai from 'sinon-chai' + +chai.use(sinonChai) +const { expect } = chai + +import * as adminHealth from '../../../src/utils/admin-health' +import * as adminMetrics from '../../../src/utils/admin-metrics' +import * as prometheusClient from '../../../src/utils/prometheus-client' + +describe('admin-metrics', () => { + let sandbox: Sinon.SinonSandbox + let queryPrometheusInstantStub: Sinon.SinonStub + let collectAdminHealthSnapshotStub: Sinon.SinonStub + + beforeEach(() => { + adminMetrics.resetAdminMetricsSnapshotCache() + sandbox = Sinon.createSandbox() + queryPrometheusInstantStub = sandbox.stub(prometheusClient, 'queryPrometheusInstant') + collectAdminHealthSnapshotStub = sandbox.stub(adminHealth, 'collectAdminHealthSnapshot').resolves({ + status: 'ok', + uptimeSeconds: 10, + worker: { type: 'worker', index: '0' }, + database: { ok: true }, + redis: { ok: true }, + }) + }) + + afterEach(() => { + adminMetrics.resetAdminMetricsSnapshotCache() + sandbox.restore() + }) + + it('builds a snapshot from prometheus queries and dependency health', async () => { + queryPrometheusInstantStub.onCall(0).resolves(1.5) + queryPrometheusInstantStub.onCall(1).resolves(0.25) + queryPrometheusInstantStub.onCall(2).resolves(100) + queryPrometheusInstantStub.onCall(3).resolves(4) + queryPrometheusInstantStub.onCall(4).resolves(12) + queryPrometheusInstantStub.onCall(5).resolves(33.3) + queryPrometheusInstantStub.onCall(6).resolves(256) + + const snapshot = await adminMetrics.collectAdminMetricsSnapshot() + + expect(snapshot.status).to.equal('ok') + expect(snapshot.metrics).to.deep.equal({ + eventsPerSecond: 1.5, + eventsRejectedPerSecond: 0.25, + eventsAcceptedTotal: 100, + eventsRejectedTotal: 4, + activeConnections: 12, + cpuLoadPercent: 33.3, + memoryUsedMb: 256, + }) + expect(snapshot.health.database.ok).to.equal(true) + expect(snapshot.prometheus.available).to.equal(true) + expect(collectAdminHealthSnapshotStub).to.have.been.calledOnce + expect(queryPrometheusInstantStub.callCount).to.equal(7) + }) + + it('marks snapshot unavailable when prometheus returns no data', async () => { + queryPrometheusInstantStub.resolves(undefined) + + const snapshot = await adminMetrics.collectAdminMetricsSnapshot() + + expect(snapshot.status).to.equal('unavailable') + expect(snapshot.prometheus.available).to.equal(false) + expect(snapshot.metrics.eventsPerSecond).to.equal(0) + }) + + it('marks snapshot degraded when dependency health fails', async () => { + collectAdminHealthSnapshotStub.resolves({ + status: 'degraded', + uptimeSeconds: 10, + worker: { type: 'worker' }, + database: { ok: false }, + redis: { ok: true }, + }) + queryPrometheusInstantStub.onCall(0).resolves(1) + + const snapshot = await adminMetrics.collectAdminMetricsSnapshot() + + expect(snapshot.status).to.equal('degraded') + expect(snapshot.health.database.ok).to.equal(false) + }) + + it('reuses cached snapshots within the SSE interval', async () => { + queryPrometheusInstantStub.resolves(1) + + await adminMetrics.collectAdminMetricsSnapshot() + await adminMetrics.collectAdminMetricsSnapshot() + + expect(queryPrometheusInstantStub.callCount).to.equal(7) + }) + + describe('getAdminMetricsSseIntervalMs', () => { + const originalInterval = process.env.ADMIN_METRICS_SSE_INTERVAL_MS + + afterEach(() => { + if (originalInterval === undefined) { + delete process.env.ADMIN_METRICS_SSE_INTERVAL_MS + } else { + process.env.ADMIN_METRICS_SSE_INTERVAL_MS = originalInterval + } + }) + + it('defaults to 5000ms', () => { + delete process.env.ADMIN_METRICS_SSE_INTERVAL_MS + + expect(adminMetrics.getAdminMetricsSseIntervalMs()).to.equal(5000) + }) + + it('falls back when configured interval is invalid', () => { + process.env.ADMIN_METRICS_SSE_INTERVAL_MS = 'abc' + + expect(adminMetrics.getAdminMetricsSseIntervalMs()).to.equal(5000) + }) + }) +}) diff --git a/test/unit/utils/prometheus-client.spec.ts b/test/unit/utils/prometheus-client.spec.ts new file mode 100644 index 00000000..c6c8a4d7 --- /dev/null +++ b/test/unit/utils/prometheus-client.spec.ts @@ -0,0 +1,110 @@ +import axios from 'axios' +import chai from 'chai' +import Sinon from 'sinon' +import sinonChai from 'sinon-chai' + +chai.use(sinonChai) +const { expect } = chai + +import * as prometheusClient from '../../../src/utils/prometheus-client' + +describe('prometheus-client', () => { + let sandbox: Sinon.SinonSandbox + let axiosGetStub: Sinon.SinonStub + const originalPrometheusUrl = process.env.PROMETHEUS_URL + + beforeEach(() => { + sandbox = Sinon.createSandbox() + axiosGetStub = sandbox.stub(axios, 'get') + delete process.env.PROMETHEUS_URL + }) + + afterEach(() => { + sandbox.restore() + if (originalPrometheusUrl === undefined) { + delete process.env.PROMETHEUS_URL + } else { + process.env.PROMETHEUS_URL = originalPrometheusUrl + } + }) + + describe('getPrometheusBaseUrl', () => { + it('defaults to localhost prometheus', () => { + expect(prometheusClient.getPrometheusBaseUrl()).to.equal('http://127.0.0.1:9090') + }) + + it('trims trailing slashes from configured URL', () => { + process.env.PROMETHEUS_URL = 'http://prometheus:9090/' + + expect(prometheusClient.getPrometheusBaseUrl()).to.equal('http://prometheus:9090') + }) + }) + + describe('parsePrometheusInstantQueryScalar', () => { + it('returns parsed scalar from successful vector response', () => { + const value = prometheusClient.parsePrometheusInstantQueryScalar({ + status: 'success', + data: { + resultType: 'vector', + result: [{ value: [1710000000, '12.5'] }], + }, + }) + + expect(value).to.equal(12.5) + }) + + it('returns parsed scalar from successful scalar response', () => { + const value = prometheusClient.parsePrometheusInstantQueryScalar({ + status: 'success', + data: { + resultType: 'scalar', + result: [1710000000, '9'], + }, + }) + + expect(value).to.equal(9) + }) + + it('returns undefined for unsuccessful responses', () => { + expect( + prometheusClient.parsePrometheusInstantQueryScalar({ + status: 'error', + error: 'bad_data', + }), + ).to.be.undefined + }) + }) + + describe('queryPrometheusInstant', () => { + it('queries prometheus instant API and returns scalar', async () => { + axiosGetStub.resolves({ + status: 200, + data: { + status: 'success', + data: { + resultType: 'vector', + result: [{ value: [1710000000, '3'] }], + }, + }, + }) + + const value = await prometheusClient.queryPrometheusInstant('sum(up)') + + expect(value).to.equal(3) + expect(axiosGetStub).to.have.been.calledOnceWith( + 'http://127.0.0.1:9090/api/v1/query', + Sinon.match({ + params: { query: 'sum(up)' }, + }), + ) + }) + + it('returns undefined when prometheus responds with non-200', async () => { + axiosGetStub.resolves({ status: 503, data: {} }) + + const value = await prometheusClient.queryPrometheusInstant('sum(up)') + + expect(value).to.be.undefined + }) + }) +})