f32ff701e1
Run Tests on Branches / Detect Changes (push) Successful in 14s
Run Tests on Branches / Backend Tests (push) Successful in 4m3s
Run Tests on Branches / Frontend Tests (push) Has been skipped
Run Tests on Branches / Frontend Mobile Tests (push) Has been skipped
Run Tests on Branches / Parapharmacy API Tests (push) Has been skipped
Run Tests on Branches / PIP Platform Tests (push) Has been skipped
- Fix OTel custom metrics not reaching Prometheus: ESM static import hoisting caused metrics.getMeter() to run before sdk.start(), getting NoopMeterProvider. Changed to lazy-init pattern so instruments are created on first use after the SDK configures the real MeterProvider. - Fix Grafana dashboards job labels: OTel adds service.namespace prefix (job=farmafinder/farmafinder-backend), updated queries to use regex match (job=~".*farmafinder-backend"). - Fix histogram metric names: OTel appends unit suffix (_milliseconds) to histogram names (e.g. http_request_duration_ms_milliseconds_bucket). Verified: app_heartbeat_total, http_requests_total, db_query_duration_ms, redis_cmd_duration_ms, and all other custom metrics now flow correctly through OTLP -> Alloy -> Prometheus remote_write.
148 lines
5.1 KiB
JavaScript
148 lines
5.1 KiB
JavaScript
// OpenTelemetry metrics for the FarmaFinder backend.
|
|
//
|
|
// IMPORTANT: In ESM, static imports are hoisted and evaluated BEFORE the
|
|
// module body runs. Since server.js does `await import('./src/tracing.js')`
|
|
// to call sdk.start(), but statically imports this file, the meter MUST be
|
|
// lazily initialised — otherwise metrics.getMeter() runs before the SDK
|
|
// configures the global MeterProvider and we get a NoopMeter.
|
|
//
|
|
// Metrics flow OTLP → Alloy → Prometheus remote_write, exactly like the
|
|
// tracing pipeline, so no extra Prometheus scrape job is required.
|
|
import { metrics } from '@opentelemetry/api';
|
|
|
|
let _meter;
|
|
|
|
function getMeter() {
|
|
if (!_meter) {
|
|
_meter = metrics.getMeter('farmafinder-backend', '1.0.0');
|
|
}
|
|
return _meter;
|
|
}
|
|
|
|
// Lazy instrument wrappers — each property getter creates the instrument on
|
|
// first access, by which point sdk.start() has run and the real MeterProvider
|
|
// is in place. The returned instruments are plain OTel Counter / Histogram
|
|
// objects, so all callers (appMetrics.foo.add(1)) work unchanged.
|
|
|
|
function lazyCounter(name, opts) {
|
|
let inst;
|
|
return {
|
|
get current() {
|
|
if (!inst) inst = getMeter().createCounter(name, opts);
|
|
return inst;
|
|
},
|
|
add(value, attrs) { this.current.add(value, attrs); },
|
|
};
|
|
}
|
|
|
|
function lazyHistogram(name, opts) {
|
|
let inst;
|
|
return {
|
|
get current() {
|
|
if (!inst) inst = getMeter().createHistogram(name, opts);
|
|
return inst;
|
|
},
|
|
record(value, attrs) { this.current.record(value, attrs); },
|
|
};
|
|
}
|
|
|
|
// --- Heartbeat -------------------------------------------------------------
|
|
// Emitted continuously so we can alert on "backend not sending telemetry"
|
|
// (remote_write has no `up` metric).
|
|
export const heartbeatTotal = lazyCounter('app_heartbeat_total', {
|
|
description: 'Heartbeat ticks from the backend process (liveness signal).',
|
|
});
|
|
|
|
// --- Medicine search / CIMA upstream --------------------------------------
|
|
export const medicineSearchesTotal = lazyCounter('medicine_searches_total', {
|
|
description: 'Total medicine searches handled (cache or upstream).',
|
|
});
|
|
|
|
export const cimaRequestsTotal = lazyCounter('cima_requests_total', {
|
|
description: 'Total upstream CIMA API requests.',
|
|
});
|
|
|
|
export const cimaRequestDuration = lazyHistogram('cima_request_duration_ms', {
|
|
description: 'Duration of upstream CIMA API requests.',
|
|
unit: 'ms',
|
|
});
|
|
|
|
export const cacheHitsTotal = lazyCounter('cache_hits_total', {
|
|
description: 'Total Redis cache hits.',
|
|
});
|
|
|
|
export const cacheMissesTotal = lazyCounter('cache_misses_total', {
|
|
description: 'Total Redis cache misses (fell through to upstream).',
|
|
});
|
|
|
|
// --- Auth ------------------------------------------------------------------
|
|
export const loginSuccessTotal = lazyCounter('login_success_total', {
|
|
description: 'Successful login attempts.',
|
|
});
|
|
|
|
export const loginFailureTotal = lazyCounter('login_failure_total', {
|
|
description: 'Failed login attempts.',
|
|
});
|
|
|
|
export const rateLimitRejectedTotal = lazyCounter('rate_limit_rejected_total', {
|
|
description: 'Requests rejected by a rate limiter.',
|
|
});
|
|
|
|
// --- Admin operations ------------------------------------------------------
|
|
export const pharmacyWriteTotal = lazyCounter('pharmacy_write_total', {
|
|
description: 'Admin pharmacy write operations.',
|
|
});
|
|
|
|
export const pharmacyMedicineLinkTotal = lazyCounter('pharmacy_medicine_link_total', {
|
|
description: 'Pharmacy-medicine link operations.',
|
|
});
|
|
|
|
// --- Push notifications ----------------------------------------------------
|
|
export const pushSentTotal = lazyCounter('push_sent_total', {
|
|
description: 'Push notifications successfully sent.',
|
|
});
|
|
|
|
export const pushFailedTotal = lazyCounter('push_failed_total', {
|
|
description: 'Push notifications that failed to send.',
|
|
});
|
|
|
|
// --- Datastores ------------------------------------------------------------
|
|
export const dbQueryDuration = lazyHistogram('db_query_duration_ms', {
|
|
description: 'Duration of user DB queries.',
|
|
unit: 'ms',
|
|
});
|
|
|
|
export const dbErrorsTotal = lazyCounter('db_errors_total', {
|
|
description: 'User DB query errors.',
|
|
});
|
|
|
|
export const redisErrorsTotal = lazyCounter('redis_errors_total', {
|
|
description: 'Redis client errors.',
|
|
});
|
|
|
|
export const redisCmdDuration = lazyHistogram('redis_cmd_duration_ms', {
|
|
description: 'Duration of Redis commands.',
|
|
unit: 'ms',
|
|
});
|
|
|
|
// --- HTTP (explicit, so we get status class + per-route breakdowns) --------
|
|
// OTel's http auto-instrumentation records duration but not a discrete
|
|
// error count, so we track requests explicitly for error-rate alerting.
|
|
export const httpRequestsTotal = lazyCounter('http_requests_total', {
|
|
description: 'Total HTTP requests, labelled by route and status class.',
|
|
});
|
|
|
|
export const httpRequestDuration = lazyHistogram('http_request_duration_ms', {
|
|
description: 'HTTP request duration.',
|
|
unit: 'ms',
|
|
});
|
|
|
|
// --- Heartbeat loop (guarded against --watch duplicate intervals) ----------
|
|
const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
if (process.env.NODE_ENV !== 'test' && !globalThis.__farmafinderHeartbeat) {
|
|
globalThis.__farmafinderHeartbeat = setInterval(() => {
|
|
heartbeatTotal.add(1);
|
|
}, HEARTBEAT_INTERVAL_MS);
|
|
globalThis.__farmafinderHeartbeat?.unref?.();
|
|
}
|