fix(otel): lazy-init metrics and fix Grafana dashboards

- Fix OTel custom metrics not reaching Prometheus: ESM static import
  hoisting caused metrics.getMeter() to run before sdk.start(), getting
  NoopMeterProvider. Changed to lazy-init pattern so instruments are
  created on first use after the SDK configures the real MeterProvider.

- Fix Grafana dashboards job labels: OTel adds service.namespace prefix
  (job=farmafinder/farmafinder-backend), updated queries to use regex
  match (job=~".*farmafinder-backend").

- Fix histogram metric names: OTel appends unit suffix (_milliseconds)
  to histogram names (e.g. http_request_duration_ms_milliseconds_bucket).

Verified: app_heartbeat_total, http_requests_total, db_query_duration_ms,
redis_cmd_duration_ms, and all other custom metrics now flow correctly
through OTLP -> Alloy -> Prometheus remote_write.

apps/backend/src/metrics.js

@@ -1,87 +1,126 @@
 // OpenTelemetry metrics for the FarmaFinder backend.
 //
-// Uses the global MeterProvider configured in `tracing.js` (imported first in
-// server.js). Metrics flow OTLP → Alloy → Prometheus remote_write, exactly like
-// the tracing pipeline, so no extra Prometheus scrape job is required.
+// IMPORTANT: In ESM, static imports are hoisted and evaluated BEFORE the
+// module body runs.  Since server.js does `await import('./src/tracing.js')`
+// to call sdk.start(), but statically imports this file, the meter MUST be
+// lazily initialised — otherwise metrics.getMeter() runs before the SDK
+// configures the global MeterProvider and we get a NoopMeter.
+//
+// Metrics flow OTLP → Alloy → Prometheus remote_write, exactly like the
+// tracing pipeline, so no extra Prometheus scrape job is required.
 import { metrics } from '@opentelemetry/api';
 
-const meter = metrics.getMeter('farmafinder-backend', '1.0.0');
+let _meter;
+
+function getMeter() {
+  if (!_meter) {
+    _meter = metrics.getMeter('farmafinder-backend', '1.0.0');
+  }
+  return _meter;
+}
+
+// Lazy instrument wrappers — each property getter creates the instrument on
+// first access, by which point sdk.start() has run and the real MeterProvider
+// is in place.  The returned instruments are plain OTel Counter / Histogram
+// objects, so all callers (appMetrics.foo.add(1)) work unchanged.
+
+function lazyCounter(name, opts) {
+  let inst;
+  return {
+    get current() {
+      if (!inst) inst = getMeter().createCounter(name, opts);
+      return inst;
+    },
+    add(value, attrs) { this.current.add(value, attrs); },
+  };
+}
+
+function lazyHistogram(name, opts) {
+  let inst;
+  return {
+    get current() {
+      if (!inst) inst = getMeter().createHistogram(name, opts);
+      return inst;
+    },
+    record(value, attrs) { this.current.record(value, attrs); },
+  };
+}
 
 // --- Heartbeat -------------------------------------------------------------
 // Emitted continuously so we can alert on "backend not sending telemetry"
 // (remote_write has no `up` metric).
-export const heartbeatTotal = meter.createCounter('app_heartbeat_total', {
+export const heartbeatTotal = lazyCounter('app_heartbeat_total', {
   description: 'Heartbeat ticks from the backend process (liveness signal).',
 });
 
 // --- Medicine search / CIMA upstream --------------------------------------
-export const medicineSearchesTotal = meter.createCounter('medicine_searches_total', {
+export const medicineSearchesTotal = lazyCounter('medicine_searches_total', {
   description: 'Total medicine searches handled (cache or upstream).',
 });
 
-export const cimaRequestsTotal = meter.createCounter('cima_requests_total', {
+export const cimaRequestsTotal = lazyCounter('cima_requests_total', {
   description: 'Total upstream CIMA API requests.',
 });
 
-export const cimaRequestDuration = meter.createHistogram('cima_request_duration_ms', {
+export const cimaRequestDuration = lazyHistogram('cima_request_duration_ms', {
   description: 'Duration of upstream CIMA API requests.',
   unit: 'ms',
 });
 
-export const cacheHitsTotal = meter.createCounter('cache_hits_total', {
+export const cacheHitsTotal = lazyCounter('cache_hits_total', {
   description: 'Total Redis cache hits.',
 });
 
-export const cacheMissesTotal = meter.createCounter('cache_misses_total', {
+export const cacheMissesTotal = lazyCounter('cache_misses_total', {
   description: 'Total Redis cache misses (fell through to upstream).',
 });
 
 // --- Auth ------------------------------------------------------------------
-export const loginSuccessTotal = meter.createCounter('login_success_total', {
+export const loginSuccessTotal = lazyCounter('login_success_total', {
   description: 'Successful login attempts.',
 });
 
-export const loginFailureTotal = meter.createCounter('login_failure_total', {
+export const loginFailureTotal = lazyCounter('login_failure_total', {
   description: 'Failed login attempts.',
 });
 
-export const rateLimitRejectedTotal = meter.createCounter('rate_limit_rejected_total', {
+export const rateLimitRejectedTotal = lazyCounter('rate_limit_rejected_total', {
   description: 'Requests rejected by a rate limiter.',
 });
 
 // --- Admin operations ------------------------------------------------------
-export const pharmacyWriteTotal = meter.createCounter('pharmacy_write_total', {
+export const pharmacyWriteTotal = lazyCounter('pharmacy_write_total', {
   description: 'Admin pharmacy write operations.',
 });
 
-export const pharmacyMedicineLinkTotal = meter.createCounter('pharmacy_medicine_link_total', {
+export const pharmacyMedicineLinkTotal = lazyCounter('pharmacy_medicine_link_total', {
   description: 'Pharmacy-medicine link operations.',
 });
 
 // --- Push notifications ----------------------------------------------------
-export const pushSentTotal = meter.createCounter('push_sent_total', {
+export const pushSentTotal = lazyCounter('push_sent_total', {
   description: 'Push notifications successfully sent.',
 });
 
-export const pushFailedTotal = meter.createCounter('push_failed_total', {
+export const pushFailedTotal = lazyCounter('push_failed_total', {
   description: 'Push notifications that failed to send.',
 });
 
 // --- Datastores ------------------------------------------------------------
-export const dbQueryDuration = meter.createHistogram('db_query_duration_ms', {
+export const dbQueryDuration = lazyHistogram('db_query_duration_ms', {
   description: 'Duration of user DB queries.',
   unit: 'ms',
 });
 
-export const dbErrorsTotal = meter.createCounter('db_errors_total', {
+export const dbErrorsTotal = lazyCounter('db_errors_total', {
   description: 'User DB query errors.',
 });
 
-export const redisErrorsTotal = meter.createCounter('redis_errors_total', {
+export const redisErrorsTotal = lazyCounter('redis_errors_total', {
   description: 'Redis client errors.',
 });
 
-export const redisCmdDuration = meter.createHistogram('redis_cmd_duration_ms', {
+export const redisCmdDuration = lazyHistogram('redis_cmd_duration_ms', {
   description: 'Duration of Redis commands.',
   unit: 'ms',
 });
@@ -89,11 +128,11 @@ export const redisCmdDuration = meter.createHistogram('redis_cmd_duration_ms', {
 // --- HTTP (explicit, so we get status class + per-route breakdowns) --------
 // OTel's http auto-instrumentation records duration but not a discrete
 // error count, so we track requests explicitly for error-rate alerting.
-export const httpRequestsTotal = meter.createCounter('http_requests_total', {
+export const httpRequestsTotal = lazyCounter('http_requests_total', {
   description: 'Total HTTP requests, labelled by route and status class.',
 });
 
-export const httpRequestDuration = meter.createHistogram('http_request_duration_ms', {
+export const httpRequestDuration = lazyHistogram('http_request_duration_ms', {
   description: 'HTTP request duration.',
   unit: 'ms',
 });