feat: add end-to-end observability (metrics, health, mobile RUM, dashboards, alerts)

- Backend: OTel metrics via OTLP -> Alloy -> Prometheus (OTEL_METRICS_EXPORTER=otlp)
- New business metrics (src/metrics.js): searches, CIMA latency/errors, cache
  hits/misses, logins, rate-limits, pharmacy writes/links, push sent/failed,
  DB + Redis timings/errors, HTTP req count/duration, heartbeat
- Backend health endpoints /healthz and /readyz
- Mobile (Expo): Grafana Faro RUM via @grafana/faro-react-native
- redis/postgres exporters in docker-compose + Prometheus scrape jobs
- Grafana dashboards (backend, datastores, mobile RUM, overview)
- Prometheus alert rules (farmafinder_*) -> existing Alertmanager (Telegram)
- Design/spec saved to docs/superpowers/specs/

apps/backend/src/metrics.js

@@ -0,0 +1,108 @@
+// OpenTelemetry metrics for the FarmaFinder backend.
+//
+// Uses the global MeterProvider configured in `tracing.js` (imported first in
+// server.js). Metrics flow OTLP → Alloy → Prometheus remote_write, exactly like
+// the tracing pipeline, so no extra Prometheus scrape job is required.
+import { metrics } from '@opentelemetry/api';
+
+const meter = metrics.getMeter('farmafinder-backend', '1.0.0');
+
+// --- Heartbeat -------------------------------------------------------------
+// Emitted continuously so we can alert on "backend not sending telemetry"
+// (remote_write has no `up` metric).
+export const heartbeatTotal = meter.createCounter('app_heartbeat_total', {
+  description: 'Heartbeat ticks from the backend process (liveness signal).',
+});
+
+// --- Medicine search / CIMA upstream --------------------------------------
+export const medicineSearchesTotal = meter.createCounter('medicine_searches_total', {
+  description: 'Total medicine searches handled (cache or upstream).',
+});
+
+export const cimaRequestsTotal = meter.createCounter('cima_requests_total', {
+  description: 'Total upstream CIMA API requests.',
+});
+
+export const cimaRequestDuration = meter.createHistogram('cima_request_duration_ms', {
+  description: 'Duration of upstream CIMA API requests.',
+  unit: 'ms',
+});
+
+export const cacheHitsTotal = meter.createCounter('cache_hits_total', {
+  description: 'Total Redis cache hits.',
+});
+
+export const cacheMissesTotal = meter.createCounter('cache_misses_total', {
+  description: 'Total Redis cache misses (fell through to upstream).',
+});
+
+// --- Auth ------------------------------------------------------------------
+export const loginSuccessTotal = meter.createCounter('login_success_total', {
+  description: 'Successful login attempts.',
+});
+
+export const loginFailureTotal = meter.createCounter('login_failure_total', {
+  description: 'Failed login attempts.',
+});
+
+export const rateLimitRejectedTotal = meter.createCounter('rate_limit_rejected_total', {
+  description: 'Requests rejected by a rate limiter.',
+});
+
+// --- Admin operations ------------------------------------------------------
+export const pharmacyWriteTotal = meter.createCounter('pharmacy_write_total', {
+  description: 'Admin pharmacy write operations.',
+});
+
+export const pharmacyMedicineLinkTotal = meter.createCounter('pharmacy_medicine_link_total', {
+  description: 'Pharmacy-medicine link operations.',
+});
+
+// --- Push notifications ----------------------------------------------------
+export const pushSentTotal = meter.createCounter('push_sent_total', {
+  description: 'Push notifications successfully sent.',
+});
+
+export const pushFailedTotal = meter.createCounter('push_failed_total', {
+  description: 'Push notifications that failed to send.',
+});
+
+// --- Datastores ------------------------------------------------------------
+export const dbQueryDuration = meter.createHistogram('db_query_duration_ms', {
+  description: 'Duration of user DB queries.',
+  unit: 'ms',
+});
+
+export const dbErrorsTotal = meter.createCounter('db_errors_total', {
+  description: 'User DB query errors.',
+});
+
+export const redisErrorsTotal = meter.createCounter('redis_errors_total', {
+  description: 'Redis client errors.',
+});
+
+export const redisCmdDuration = meter.createHistogram('redis_cmd_duration_ms', {
+  description: 'Duration of Redis commands.',
+  unit: 'ms',
+});
+
+// --- HTTP (explicit, so we get status class + per-route breakdowns) --------
+// OTel's http auto-instrumentation records duration but not a discrete
+// error count, so we track requests explicitly for error-rate alerting.
+export const httpRequestsTotal = meter.createCounter('http_requests_total', {
+  description: 'Total HTTP requests, labelled by route and status class.',
+});
+
+export const httpRequestDuration = meter.createHistogram('http_request_duration_ms', {
+  description: 'HTTP request duration.',
+  unit: 'ms',
+});
+
+// --- Heartbeat loop (guarded against --watch duplicate intervals) ----------
+const HEARTBEAT_INTERVAL_MS = 30_000;
+if (process.env.NODE_ENV !== 'test' && !globalThis.__farmafinderHeartbeat) {
+  globalThis.__farmafinderHeartbeat = setInterval(() => {
+    heartbeatTotal.add(1);
+  }, HEARTBEAT_INTERVAL_MS);
+  globalThis.__farmafinderHeartbeat?.unref?.();
+}

apps/backend/src/tracing.js

@@ -9,8 +9,10 @@
 // routed to Tempo.
 
 import { NodeSDK } from '@opentelemetry/sdk-node';
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
 import * as resources from '@opentelemetry/resources';
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_NAMESPACE } from '@opentelemetry/semantic-conventions';
 import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino';
@@ -28,9 +30,18 @@ const resource = typeof resources.resourceFromAttributes === 'function'
       [ATTR_SERVICE_NAMESPACE]: 'farmaclic',
     });
 
+// Metrics → OTLP → Alloy → Prometheus remote_write. The PeriodicExportingMetricReader
+// makes the auto-instrumentations (http/express/redis) emit request duration/error
+// metrics with no extra code, on top of the custom business metrics in src/metrics.js.
+const metricReader = new PeriodicExportingMetricReader({
+  exportIntervalMillis: 15_000,
+  exporter: new OTLPMetricExporter({ url: otlpEndpoint }),
+});
+
 const sdk = new NodeSDK({
   resource,
   traceExporter: new OTLPTraceExporter({ url: otlpEndpoint }),
+  metricReader,
   instrumentations: [
     getNodeAutoInstrumentations({
       // Disable fs by default — it is noisy and rarely useful.