feat: add end-to-end observability (metrics, health, mobile RUM, dashboards, alerts)
- Backend: OTel metrics via OTLP -> Alloy -> Prometheus (OTEL_METRICS_EXPORTER=otlp) - New business metrics (src/metrics.js): searches, CIMA latency/errors, cache hits/misses, logins, rate-limits, pharmacy writes/links, push sent/failed, DB + Redis timings/errors, HTTP req count/duration, heartbeat - Backend health endpoints /healthz and /readyz - Mobile (Expo): Grafana Faro RUM via @grafana/faro-react-native - redis/postgres exporters in docker-compose + Prometheus scrape jobs - Grafana dashboards (backend, datastores, mobile RUM, overview) - Prometheus alert rules (farmafinder_*) -> existing Alertmanager (Telegram) - Design/spec saved to docs/superpowers/specs/
This commit is contained in:
Antoni Nuñez Romeu
+102
-16
@@ -2,6 +2,8 @@
|
||||
import './src/tracing.js';
|
||||
|
||||
import express from 'express';
|
||||
import redisClient from './redis-client.js';
|
||||
import * as appMetrics from './src/metrics.js';
|
||||
import cors from 'cors';
|
||||
import sqlite3 from 'sqlite3';
|
||||
import { promisify } from 'util';
|
||||
@@ -56,6 +58,19 @@ app.use(cors({
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
app.use(pinoHttp({ logger }));
|
||||
|
||||
// Explicit HTTP request metrics (count by route + status class, duration).
|
||||
// Auto-instrumentation gives duration histograms but no discrete error count.
|
||||
app.use((req, res, next) => {
|
||||
const start = performance.now();
|
||||
res.on('finish', () => {
|
||||
const route = req.route?.path || req.path;
|
||||
const statusClass = `${Math.floor(res.statusCode / 100)}xx`;
|
||||
appMetrics.httpRequestsTotal.add(1, { route, status_class: statusClass });
|
||||
appMetrics.httpRequestDuration.record(performance.now() - start, { route });
|
||||
});
|
||||
next();
|
||||
});
|
||||
|
||||
const PG_URL = process.env.PG_URL;
|
||||
let pgPool = null;
|
||||
if (PG_URL) {
|
||||
@@ -88,10 +103,16 @@ if (process.env.NODE_ENV !== 'test') {
|
||||
// Configure session
|
||||
app.use(session(sessionConfig));
|
||||
|
||||
const searchLimiter = rateLimit({ windowMs: 60_000, max: 30, standardHeaders: true, legacyHeaders: false });
|
||||
const loginLimiter = rateLimit({ windowMs: 60_000, max: 5, standardHeaders: true, legacyHeaders: false });
|
||||
const registerLimiter = rateLimit({ windowMs: 60 * 60_000, max: 10, standardHeaders: true, legacyHeaders: false });
|
||||
const geocodeLimiter = rateLimit({ windowMs: 60_000, max: 10, standardHeaders: true, legacyHeaders: false });
|
||||
// Shared handler so every limiter emits a counter (route label) on rejection.
|
||||
const limitHandler = (route) => (req, res, next) => {
|
||||
appMetrics.rateLimitRejectedTotal.add(1, { route });
|
||||
res.status(429).json({ error: 'Too many requests, please try again later.' });
|
||||
};
|
||||
|
||||
const searchLimiter = rateLimit({ windowMs: 60_000, max: 30, standardHeaders: true, legacyHeaders: false, handler: limitHandler('search') });
|
||||
const loginLimiter = rateLimit({ windowMs: 60_000, max: 5, standardHeaders: true, legacyHeaders: false, handler: limitHandler('login') });
|
||||
const registerLimiter = rateLimit({ windowMs: 60 * 60_000, max: 10, standardHeaders: true, legacyHeaders: false, handler: limitHandler('register') });
|
||||
const geocodeLimiter = rateLimit({ windowMs: 60_000, max: 10, standardHeaders: true, legacyHeaders: false, handler: limitHandler('geocode') });
|
||||
|
||||
const VAPID_PUBLIC_KEY = process.env.VAPID_PUBLIC_KEY || '';
|
||||
const VAPID_PRIVATE_KEY = process.env.VAPID_PRIVATE_KEY || '';
|
||||
@@ -128,27 +149,54 @@ function toPositional(sql) {
|
||||
}
|
||||
|
||||
async function userDbGet(sql, params = []) {
|
||||
if (pgPool) {
|
||||
const res = await pgPool.query(toPositional(sql), params);
|
||||
return res.rows[0] ?? null;
|
||||
const engine = pgPool ? 'pg' : 'sqlite';
|
||||
const start = performance.now();
|
||||
try {
|
||||
if (pgPool) {
|
||||
const res = await pgPool.query(toPositional(sql), params);
|
||||
return res.rows[0] ?? null;
|
||||
}
|
||||
return await dbGet(sql, params);
|
||||
} catch (err) {
|
||||
appMetrics.dbErrorsTotal.add(1, { engine });
|
||||
throw err;
|
||||
} finally {
|
||||
appMetrics.dbQueryDuration.record(performance.now() - start, { engine });
|
||||
}
|
||||
return dbGet(sql, params);
|
||||
}
|
||||
|
||||
async function userDbRun(sql, params = []) {
|
||||
if (pgPool) {
|
||||
const res = await pgPool.query(toPositional(sql), params);
|
||||
return { lastID: res.rows[0]?.id, changes: res.rowCount };
|
||||
const engine = pgPool ? 'pg' : 'sqlite';
|
||||
const start = performance.now();
|
||||
try {
|
||||
if (pgPool) {
|
||||
const res = await pgPool.query(toPositional(sql), params);
|
||||
return { lastID: res.rows[0]?.id, changes: res.rowCount };
|
||||
}
|
||||
return await dbRun(sql.replace(/\s+RETURNING\s+\w+\s*$/i, ''), params);
|
||||
} catch (err) {
|
||||
appMetrics.dbErrorsTotal.add(1, { engine });
|
||||
throw err;
|
||||
} finally {
|
||||
appMetrics.dbQueryDuration.record(performance.now() - start, { engine });
|
||||
}
|
||||
return dbRun(sql.replace(/\s+RETURNING\s+\w+\s*$/i, ''), params);
|
||||
}
|
||||
|
||||
async function userDbAll(sql, params = []) {
|
||||
if (pgPool) {
|
||||
const res = await pgPool.query(toPositional(sql), params);
|
||||
return res.rows;
|
||||
const engine = pgPool ? 'pg' : 'sqlite';
|
||||
const start = performance.now();
|
||||
try {
|
||||
if (pgPool) {
|
||||
const res = await pgPool.query(toPositional(sql), params);
|
||||
return res.rows;
|
||||
}
|
||||
return await dbAll(sql, params);
|
||||
} catch (err) {
|
||||
appMetrics.dbErrorsTotal.add(1, { engine });
|
||||
throw err;
|
||||
} finally {
|
||||
appMetrics.dbQueryDuration.record(performance.now() - start, { engine });
|
||||
}
|
||||
return dbAll(sql, params);
|
||||
}
|
||||
|
||||
// Accept opening_hours as either an object or a JSON string from the client
|
||||
@@ -861,6 +909,7 @@ app.post('/api/auth/login', loginLimiter, async (req, res) => {
|
||||
);
|
||||
|
||||
if (!user) {
|
||||
appMetrics.loginFailureTotal.add(1);
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
|
||||
@@ -868,6 +917,7 @@ app.post('/api/auth/login', loginLimiter, async (req, res) => {
|
||||
const isValidPassword = await bcrypt.compare(password, user.password_hash);
|
||||
|
||||
if (!isValidPassword) {
|
||||
appMetrics.loginFailureTotal.add(1);
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
|
||||
@@ -876,6 +926,7 @@ app.post('/api/auth/login', loginLimiter, async (req, res) => {
|
||||
req.session.username = user.username;
|
||||
req.session.isAdmin = Boolean(user.is_admin);
|
||||
|
||||
appMetrics.loginSuccessTotal.add(1);
|
||||
res.json({
|
||||
message: 'Login successful',
|
||||
user: {
|
||||
@@ -1583,6 +1634,7 @@ app.post('/api/admin/pharmacies', requireAdmin, async (req, res) => {
|
||||
}
|
||||
|
||||
res.status(201).json(newPharmacy);
|
||||
appMetrics.pharmacyWriteTotal.add(1, { op: 'create' });
|
||||
} catch (error) {
|
||||
console.error('Error adding pharmacy:', error);
|
||||
if (error.message.includes('UNIQUE constraint')) {
|
||||
@@ -1620,6 +1672,7 @@ app.put('/api/admin/pharmacies/:id', requireAdmin, async (req, res) => {
|
||||
}
|
||||
|
||||
res.json(updatedPharmacy);
|
||||
appMetrics.pharmacyWriteTotal.add(1, { op: 'update' });
|
||||
} catch (error) {
|
||||
console.error('Error updating pharmacy:', error);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
@@ -1638,6 +1691,7 @@ app.delete('/api/admin/pharmacies/:id', requireAdmin, async (req, res) => {
|
||||
await userDbRun('DELETE FROM pharmacies WHERE id = ?', [pharmacyId]);
|
||||
|
||||
res.json({ message: 'Pharmacy deleted successfully' });
|
||||
appMetrics.pharmacyWriteTotal.add(1, { op: 'delete' });
|
||||
} catch (error) {
|
||||
console.error('Error deleting pharmacy:', error);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
@@ -1807,6 +1861,7 @@ app.post('/api/admin/pharmacy-medicines', requireAdmin, async (req, res) => {
|
||||
}
|
||||
|
||||
res.status(201).json(relationship);
|
||||
appMetrics.pharmacyMedicineLinkTotal.add(1, { op: 'create' });
|
||||
} catch (error) {
|
||||
console.error('Error adding medicine to pharmacy:', error);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
@@ -1834,6 +1889,7 @@ app.put('/api/admin/pharmacy-medicines/:id', requireAdmin, async (req, res) => {
|
||||
}
|
||||
|
||||
res.json(updated);
|
||||
appMetrics.pharmacyMedicineLinkTotal.add(1, { op: 'update' });
|
||||
} catch (error) {
|
||||
console.error('Error updating pharmacy-medicine:', error);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
@@ -1846,6 +1902,7 @@ app.delete('/api/admin/pharmacy-medicines/:id', requireAdmin, async (req, res) =
|
||||
const id = parseInt(req.params.id);
|
||||
await userDbRun('DELETE FROM pharmacy_medicines WHERE id = ?', [id]);
|
||||
res.json({ message: 'Medicine removed from pharmacy successfully' });
|
||||
appMetrics.pharmacyMedicineLinkTotal.add(1, { op: 'delete' });
|
||||
} catch (error) {
|
||||
console.error('Error deleting pharmacy-medicine:', error);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
@@ -2157,8 +2214,10 @@ async function sendPushForMedicine({ medicine_nregistro, medicine_name, pharmacy
|
||||
payload
|
||||
);
|
||||
sent++;
|
||||
appMetrics.pushSentTotal.add(1, { channel: 'web' });
|
||||
} catch (err) {
|
||||
failed++;
|
||||
appMetrics.pushFailedTotal.add(1, { channel: 'web' });
|
||||
const status = err?.statusCode;
|
||||
if (status === 404 || status === 410) {
|
||||
try {
|
||||
@@ -2214,8 +2273,10 @@ async function sendPushForMedicine({ medicine_nregistro, medicine_name, pharmacy
|
||||
for (const ticket of result.data) {
|
||||
if (ticket.status === 'ok') {
|
||||
sent++;
|
||||
appMetrics.pushSentTotal.add(1, { channel: 'expo' });
|
||||
} else {
|
||||
failed++;
|
||||
appMetrics.pushFailedTotal.add(1, { channel: 'expo' });
|
||||
// Prune invalid tokens
|
||||
if (ticket.message?.includes('InvalidCredentials') || ticket.message?.includes('DeviceNotRegistered')) {
|
||||
const expoEndpoint = `expo://${ticket.id || ''}`;
|
||||
@@ -2230,13 +2291,16 @@ async function sendPushForMedicine({ medicine_nregistro, medicine_name, pharmacy
|
||||
}
|
||||
} else {
|
||||
sent += chunk.length;
|
||||
appMetrics.pushSentTotal.add(chunk.length, { channel: 'expo' });
|
||||
}
|
||||
} else {
|
||||
failed += chunk.length;
|
||||
appMetrics.pushFailedTotal.add(chunk.length, { channel: 'expo' });
|
||||
console.error('[push] expo send failed:', response.status);
|
||||
}
|
||||
} catch (err) {
|
||||
failed += chunk.length;
|
||||
appMetrics.pushFailedTotal.add(chunk.length, { channel: 'expo' });
|
||||
console.error('[push] expo send error:', err.message);
|
||||
}
|
||||
}
|
||||
@@ -2245,6 +2309,28 @@ async function sendPushForMedicine({ medicine_nregistro, medicine_name, pharmacy
|
||||
return { sent, failed, pruned };
|
||||
}
|
||||
|
||||
// --- Health & readiness probes --------------------------------------------
|
||||
// /healthz: liveness (process is up). /readyz: dependencies reachable.
|
||||
app.get('/healthz', (_req, res) => {
|
||||
res.json({ status: 'ok', uptime: process.uptime() });
|
||||
});
|
||||
|
||||
app.get('/readyz', async (_req, res) => {
|
||||
const checks = { redis: 'ok', db: 'ok' };
|
||||
try {
|
||||
await redisClient.ping();
|
||||
} catch {
|
||||
checks.redis = 'fail';
|
||||
}
|
||||
try {
|
||||
await userDbGet('SELECT 1');
|
||||
} catch {
|
||||
checks.db = 'fail';
|
||||
}
|
||||
const ready = checks.redis === 'ok' && checks.db === 'ok';
|
||||
res.status(ready ? 200 : 503).json({ status: ready ? 'ok' : 'degraded', checks });
|
||||
});
|
||||
|
||||
export { app, initDatabase, db };
|
||||
|
||||
if (process.env.NODE_ENV !== 'test') {
|
||||
|
||||
Reference in New Issue
Block a user