diff --git a/apps/backend/src/metrics.js b/apps/backend/src/metrics.js index 1d052c3..80ec826 100644 --- a/apps/backend/src/metrics.js +++ b/apps/backend/src/metrics.js @@ -1,87 +1,126 @@ // OpenTelemetry metrics for the FarmaFinder backend. // -// Uses the global MeterProvider configured in `tracing.js` (imported first in -// server.js). Metrics flow OTLP → Alloy → Prometheus remote_write, exactly like -// the tracing pipeline, so no extra Prometheus scrape job is required. +// IMPORTANT: In ESM, static imports are hoisted and evaluated BEFORE the +// module body runs. Since server.js does `await import('./src/tracing.js')` +// to call sdk.start(), but statically imports this file, the meter MUST be +// lazily initialised — otherwise metrics.getMeter() runs before the SDK +// configures the global MeterProvider and we get a NoopMeter. +// +// Metrics flow OTLP → Alloy → Prometheus remote_write, exactly like the +// tracing pipeline, so no extra Prometheus scrape job is required. import { metrics } from '@opentelemetry/api'; -const meter = metrics.getMeter('farmafinder-backend', '1.0.0'); +let _meter; + +function getMeter() { + if (!_meter) { + _meter = metrics.getMeter('farmafinder-backend', '1.0.0'); + } + return _meter; +} + +// Lazy instrument wrappers — each property getter creates the instrument on +// first access, by which point sdk.start() has run and the real MeterProvider +// is in place. The returned instruments are plain OTel Counter / Histogram +// objects, so all callers (appMetrics.foo.add(1)) work unchanged. + +function lazyCounter(name, opts) { + let inst; + return { + get current() { + if (!inst) inst = getMeter().createCounter(name, opts); + return inst; + }, + add(value, attrs) { this.current.add(value, attrs); }, + }; +} + +function lazyHistogram(name, opts) { + let inst; + return { + get current() { + if (!inst) inst = getMeter().createHistogram(name, opts); + return inst; + }, + record(value, attrs) { this.current.record(value, attrs); }, + }; +} // --- Heartbeat ------------------------------------------------------------- // Emitted continuously so we can alert on "backend not sending telemetry" // (remote_write has no `up` metric). -export const heartbeatTotal = meter.createCounter('app_heartbeat_total', { +export const heartbeatTotal = lazyCounter('app_heartbeat_total', { description: 'Heartbeat ticks from the backend process (liveness signal).', }); // --- Medicine search / CIMA upstream -------------------------------------- -export const medicineSearchesTotal = meter.createCounter('medicine_searches_total', { +export const medicineSearchesTotal = lazyCounter('medicine_searches_total', { description: 'Total medicine searches handled (cache or upstream).', }); -export const cimaRequestsTotal = meter.createCounter('cima_requests_total', { +export const cimaRequestsTotal = lazyCounter('cima_requests_total', { description: 'Total upstream CIMA API requests.', }); -export const cimaRequestDuration = meter.createHistogram('cima_request_duration_ms', { +export const cimaRequestDuration = lazyHistogram('cima_request_duration_ms', { description: 'Duration of upstream CIMA API requests.', unit: 'ms', }); -export const cacheHitsTotal = meter.createCounter('cache_hits_total', { +export const cacheHitsTotal = lazyCounter('cache_hits_total', { description: 'Total Redis cache hits.', }); -export const cacheMissesTotal = meter.createCounter('cache_misses_total', { +export const cacheMissesTotal = lazyCounter('cache_misses_total', { description: 'Total Redis cache misses (fell through to upstream).', }); // --- Auth ------------------------------------------------------------------ -export const loginSuccessTotal = meter.createCounter('login_success_total', { +export const loginSuccessTotal = lazyCounter('login_success_total', { description: 'Successful login attempts.', }); -export const loginFailureTotal = meter.createCounter('login_failure_total', { +export const loginFailureTotal = lazyCounter('login_failure_total', { description: 'Failed login attempts.', }); -export const rateLimitRejectedTotal = meter.createCounter('rate_limit_rejected_total', { +export const rateLimitRejectedTotal = lazyCounter('rate_limit_rejected_total', { description: 'Requests rejected by a rate limiter.', }); // --- Admin operations ------------------------------------------------------ -export const pharmacyWriteTotal = meter.createCounter('pharmacy_write_total', { +export const pharmacyWriteTotal = lazyCounter('pharmacy_write_total', { description: 'Admin pharmacy write operations.', }); -export const pharmacyMedicineLinkTotal = meter.createCounter('pharmacy_medicine_link_total', { +export const pharmacyMedicineLinkTotal = lazyCounter('pharmacy_medicine_link_total', { description: 'Pharmacy-medicine link operations.', }); // --- Push notifications ---------------------------------------------------- -export const pushSentTotal = meter.createCounter('push_sent_total', { +export const pushSentTotal = lazyCounter('push_sent_total', { description: 'Push notifications successfully sent.', }); -export const pushFailedTotal = meter.createCounter('push_failed_total', { +export const pushFailedTotal = lazyCounter('push_failed_total', { description: 'Push notifications that failed to send.', }); // --- Datastores ------------------------------------------------------------ -export const dbQueryDuration = meter.createHistogram('db_query_duration_ms', { +export const dbQueryDuration = lazyHistogram('db_query_duration_ms', { description: 'Duration of user DB queries.', unit: 'ms', }); -export const dbErrorsTotal = meter.createCounter('db_errors_total', { +export const dbErrorsTotal = lazyCounter('db_errors_total', { description: 'User DB query errors.', }); -export const redisErrorsTotal = meter.createCounter('redis_errors_total', { +export const redisErrorsTotal = lazyCounter('redis_errors_total', { description: 'Redis client errors.', }); -export const redisCmdDuration = meter.createHistogram('redis_cmd_duration_ms', { +export const redisCmdDuration = lazyHistogram('redis_cmd_duration_ms', { description: 'Duration of Redis commands.', unit: 'ms', }); @@ -89,11 +128,11 @@ export const redisCmdDuration = meter.createHistogram('redis_cmd_duration_ms', { // --- HTTP (explicit, so we get status class + per-route breakdowns) -------- // OTel's http auto-instrumentation records duration but not a discrete // error count, so we track requests explicitly for error-rate alerting. -export const httpRequestsTotal = meter.createCounter('http_requests_total', { +export const httpRequestsTotal = lazyCounter('http_requests_total', { description: 'Total HTTP requests, labelled by route and status class.', }); -export const httpRequestDuration = meter.createHistogram('http_request_duration_ms', { +export const httpRequestDuration = lazyHistogram('http_request_duration_ms', { description: 'HTTP request duration.', unit: 'ms', }); diff --git a/monitoring/dashboards/farmafinder-backend.json b/monitoring/dashboards/farmafinder-backend.json index ce8e731..b67247a 100644 --- a/monitoring/dashboards/farmafinder-backend.json +++ b/monitoring/dashboards/farmafinder-backend.json @@ -12,7 +12,7 @@ "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum(rate(http_requests_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "req/s" } + { "expr": "sum(rate(http_requests_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "req/s" } ] }, { @@ -20,7 +20,7 @@ "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum by (status_class) (rate(http_requests_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "{{status_class}}" } + { "expr": "sum by (status_class) (rate(http_requests_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "{{status_class}}" } ] }, { @@ -28,7 +28,7 @@ "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum(rate(http_requests_total{status_class=\"5xx\",job=\"farmafinder-backend\"}[5m])) / sum(rate(http_requests_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "5xx ratio" } + { "expr": "sum(rate(http_requests_total{status_class=\"5xx\",job=~\".*farmafinder-backend\"}[5m])) / sum(rate(http_requests_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "5xx ratio" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } } } }, @@ -37,7 +37,7 @@ "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_ms_bucket{job=\"farmafinder-backend\"}[5m])))", "legendFormat": "{{route}} p95" } + { "expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_ms_milliseconds_bucket{job=~\".*farmafinder-backend\"}[5m])))", "legendFormat": "{{route}} p95" } ], "fieldConfig": { "defaults": { "unit": "ms" } } }, @@ -46,8 +46,8 @@ "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum(rate(medicine_searches_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "searches/s" }, - { "expr": "sum(rate(cima_requests_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "cima req/s" } + { "expr": "sum(rate(medicine_searches_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "searches/s" }, + { "expr": "sum(rate(cima_requests_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "cima req/s" } ] }, { @@ -55,7 +55,7 @@ "gridPos": { "h": 6, "w": 8, "x": 0, "y": 12 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum(rate(cache_hits_total{job=\"farmafinder-backend\"}[5m])) / (sum(rate(cache_hits_total{job=\"farmafinder-backend\"}[5m])) + sum(rate(cache_misses_total{job=\"farmafinder-backend\"}[5m])))", "legendFormat": "hit ratio" } + { "expr": "sum(rate(cache_hits_total{job=~\".*farmafinder-backend\"}[5m])) / (sum(rate(cache_hits_total{job=~\".*farmafinder-backend\"}[5m])) + sum(rate(cache_misses_total{job=~\".*farmafinder-backend\"}[5m])))", "legendFormat": "hit ratio" } ], "fieldConfig": { "defaults": { "unit": "percentunit" } } }, @@ -64,7 +64,7 @@ "gridPos": { "h": 6, "w": 8, "x": 8, "y": 12 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum(rate(cima_requests_total{status=\"error\",job=\"farmafinder-backend\"}[5m]))", "legendFormat": "errors/s" } + { "expr": "sum(rate(cima_requests_total{status=\"error\",job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "errors/s" } ] }, { @@ -72,8 +72,8 @@ "gridPos": { "h": 6, "w": 8, "x": 16, "y": 12 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum(rate(login_success_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "success/s" }, - { "expr": "sum(rate(login_failure_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "failure/s" } + { "expr": "sum(rate(login_success_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "success/s" }, + { "expr": "sum(rate(login_failure_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "failure/s" } ] }, { @@ -81,7 +81,7 @@ "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum by (route) (rate(rate_limit_rejected_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "{{route}}" } + { "expr": "sum by (route) (rate(rate_limit_rejected_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "{{route}}" } ] }, { @@ -89,8 +89,8 @@ "gridPos": { "h": 6, "w": 12, "x": 12, "y": 18 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum by (channel) (rate(push_sent_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "sent {{channel}}" }, - { "expr": "sum by (channel) (rate(push_failed_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "failed {{channel}}" } + { "expr": "sum by (channel) (rate(push_sent_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "sent {{channel}}" }, + { "expr": "sum by (channel) (rate(push_failed_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "failed {{channel}}" } ] }, { @@ -98,8 +98,8 @@ "gridPos": { "h": 6, "w": 12, "x": 0, "y": 24 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "histogram_quantile(0.95, sum by (le, engine) (rate(db_query_duration_ms_bucket{job=\"farmafinder-backend\"}[5m])))", "legendFormat": "p95 {{engine}}" }, - { "expr": "sum by (engine) (rate(db_errors_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "errors {{engine}}" } + { "expr": "histogram_quantile(0.95, sum by (le, engine) (rate(db_query_duration_ms_milliseconds_bucket{job=~\".*farmafinder-backend\"}[5m])))", "legendFormat": "p95 {{engine}}" }, + { "expr": "sum by (engine) (rate(db_errors_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "errors {{engine}}" } ], "fieldConfig": { "defaults": { "unit": "ms" } } }, @@ -108,8 +108,8 @@ "gridPos": { "h": 6, "w": 12, "x": 12, "y": 24 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "histogram_quantile(0.95, sum by (le) (rate(redis_cmd_duration_ms_bucket{job=\"farmafinder-backend\"}[5m])))", "legendFormat": "p95 redis" }, - { "expr": "sum(rate(redis_errors_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "redis errors/s" } + { "expr": "histogram_quantile(0.95, sum by (le) (rate(redis_cmd_duration_ms_milliseconds_bucket{job=~\".*farmafinder-backend\"}[5m])))", "legendFormat": "p95 redis" }, + { "expr": "sum(rate(redis_errors_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "redis errors/s" } ], "fieldConfig": { "defaults": { "unit": "ms" } } }, @@ -118,8 +118,8 @@ "gridPos": { "h": 6, "w": 12, "x": 0, "y": 30 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "sum by (op) (rate(pharmacy_write_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "pharmacy {{op}}" }, - { "expr": "sum by (op) (rate(pharmacy_medicine_link_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "link {{op}}" } + { "expr": "sum by (op) (rate(pharmacy_write_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "pharmacy {{op}}" }, + { "expr": "sum by (op) (rate(pharmacy_medicine_link_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "link {{op}}" } ] }, { @@ -127,7 +127,7 @@ "gridPos": { "h": 6, "w": 12, "x": 12, "y": 30 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ - { "expr": "max(app_heartbeat_total{job=\"farmafinder-backend\"})", "legendFormat": "heartbeat total" } + { "expr": "max(app_heartbeat_total{job=~\".*farmafinder-backend\"})", "legendFormat": "heartbeat total" } ] } ], @@ -140,6 +140,6 @@ "timezone": "browser", "title": "FarmaFinder — Backend", "uid": "farmafinder-backend", - "version": 1, + "version": 2, "weekStart": "" } diff --git a/monitoring/dashboards/farmafinder-overview.json b/monitoring/dashboards/farmafinder-overview.json index 32838ac..d0cfeda 100644 --- a/monitoring/dashboards/farmafinder-overview.json +++ b/monitoring/dashboards/farmafinder-overview.json @@ -11,20 +11,20 @@ "id": 1, "title": "Backend req/s", "type": "stat", "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "targets": [ { "expr": "sum(rate(http_requests_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "req/s" } ] + "targets": [ { "expr": "sum(rate(http_requests_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "req/s" } ] }, { "id": 2, "title": "Backend 5xx ratio", "type": "stat", "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "targets": [ { "expr": "sum(rate(http_requests_total{status_class=\"5xx\",job=\"farmafinder-backend\"}[5m])) / sum(rate(http_requests_total{job=\"farmafinder-backend\"}[5m]))", "legendFormat": "5xx" } ], + "targets": [ { "expr": "sum(rate(http_requests_total{status_class=\"5xx\",job=~\".*farmafinder-backend\"}[5m])) / sum(rate(http_requests_total{job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "5xx" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } } } }, { "id": 3, "title": "CIMA errors/s", "type": "stat", "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 }, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "targets": [ { "expr": "sum(rate(cima_requests_total{status=\"error\",job=\"farmafinder-backend\"}[5m]))", "legendFormat": "cima err/s" } ] + "targets": [ { "expr": "sum(rate(cima_requests_total{status=\"error\",job=~\".*farmafinder-backend\"}[5m]))", "legendFormat": "cima err/s" } ] }, { "id": 4, "title": "Mobile events/s", "type": "stat", @@ -59,6 +59,6 @@ "timezone": "browser", "title": "FarmaFinder — Overview", "uid": "farmafinder-overview", - "version": 1, + "version": 2, "weekStart": "" }