import { createRequire } from "node:module"; var __create = Object.create; var __getProtoOf = Object.getPrototypeOf; var __defProp = Object.defineProperty; var __getOwnPropNames = Object.getOwnPropertyNames; var __hasOwnProp = Object.prototype.hasOwnProperty; var __toESM = (mod, isNodeMode, target) => { target = mod != null ? __create(__getProtoOf(mod)) : {}; const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target; for (let key of __getOwnPropNames(mod)) if (!__hasOwnProp.call(to, key)) __defProp(to, key, { get: () => mod[key], enumerable: true }); return to; }; var __require = /* @__PURE__ */ createRequire(import.meta.url); // src/util/ds_error.ts import { TaggedError } from "better-result"; class DurableStreamsError extends TaggedError("DurableStreamsError")() { } function dsError(message, opts) { return new DurableStreamsError({ message, ...opts?.cause !== undefined ? { cause: opts.cause } : {}, ...opts?.code !== undefined ? { code: opts.code } : {} }); } // src/local/common.ts function readOptionValue(args, flag) { const idx = args.findIndex((arg) => arg === flag || arg.startsWith(`${flag}=`)); if (idx === -1) return { found: false }; const raw = args[idx]; if (raw.includes("=")) return { found: true, value: raw.split("=", 2)[1] }; const next = args[idx + 1]; if (!next || next.startsWith("--")) return { found: true }; return { found: true, value: next }; } function readFirstPositionalArg(args) { for (let i = 0;i < args.length; i += 1) { const arg = args[i]; if (arg === "--name" || arg === "--hostname" || arg === "--port") { i += 1; continue; } if (arg.startsWith("--name=") || arg.startsWith("--hostname=") || arg.startsWith("--port=")) { continue; } if (!arg.startsWith("--")) return arg; } return; } function parseLocalProcessOptions(args, opts = {}) { const out = {}; const name = readOptionValue(args, "--name"); if (name.found) out.name = name.value ?? "default"; else if (opts.allowPositionalName) { out.name = readFirstPositionalArg(args) ?? opts.defaultNameWhenMissing; } else if (opts.defaultNameWhenMissing != null) { out.name = opts.defaultNameWhenMissing; } const hostname = readOptionValue(args, "--hostname"); if (hostname.found) out.hostname = hostname.value ?? "127.0.0.1"; else if (opts.defaultHostnameWhenMissing != null) out.hostname = opts.defaultHostnameWhenMissing; const port = readOptionValue(args, "--port"); if (port.found) out.port = parsePortValue(port.value ?? "0"); else if (opts.defaultPortWhenMissing != null) out.port = opts.defaultPortWhenMissing; return out; } function parsePortValue(raw, flag = "--port") { const port = Number(raw); if (!Number.isFinite(port) || port < 0) { throw dsError(`invalid ${flag}: ${raw}`); } return Math.floor(port); } // src/app_core.ts import { mkdirSync } from "node:fs"; // src/db/schema.ts var SCHEMA_VERSION = 11; var DEFAULT_PRAGMAS_SQL = ` PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000; PRAGMA temp_store = MEMORY; `; var CREATE_TABLES_V4_SQL = ` CREATE TABLE IF NOT EXISTS streams ( stream TEXT PRIMARY KEY, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, content_type TEXT NOT NULL, stream_seq TEXT NULL, closed INTEGER NOT NULL DEFAULT 0, closed_producer_id TEXT NULL, closed_producer_epoch INTEGER NULL, closed_producer_seq INTEGER NULL, ttl_seconds INTEGER NULL, epoch INTEGER NOT NULL, next_offset INTEGER NOT NULL, sealed_through INTEGER NOT NULL, uploaded_through INTEGER NOT NULL, uploaded_segment_count INTEGER NOT NULL DEFAULT 0, pending_rows INTEGER NOT NULL, pending_bytes INTEGER NOT NULL, -- Logical size of retained rows in the wal table for this stream (payload-only bytes). -- This is explicitly tracked because SQLite file size is high-water and does not shrink -- deterministically after DELETE-based GC/retention trimming. wal_rows INTEGER NOT NULL DEFAULT 0, wal_bytes INTEGER NOT NULL DEFAULT 0, last_append_ms INTEGER NOT NULL, last_segment_cut_ms INTEGER NOT NULL, segment_in_progress INTEGER NOT NULL, expires_at_ms INTEGER NULL, stream_flags INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS streams_pending_bytes_idx ON streams(pending_bytes); CREATE INDEX IF NOT EXISTS streams_last_cut_idx ON streams(last_segment_cut_ms); CREATE INDEX IF NOT EXISTS streams_inprog_pending_idx ON streams(segment_in_progress, pending_bytes, last_segment_cut_ms); CREATE TABLE IF NOT EXISTS wal ( id INTEGER PRIMARY KEY, stream TEXT NOT NULL, offset INTEGER NOT NULL, ts_ms INTEGER NOT NULL, payload BLOB NOT NULL, payload_len INTEGER NOT NULL, routing_key BLOB NULL, content_type TEXT NULL, flags INTEGER NOT NULL DEFAULT 0 ); CREATE UNIQUE INDEX IF NOT EXISTS wal_stream_offset_uniq ON wal(stream, offset); CREATE INDEX IF NOT EXISTS wal_stream_offset_idx ON wal(stream, offset); CREATE INDEX IF NOT EXISTS wal_ts_idx ON wal(ts_ms); CREATE TABLE IF NOT EXISTS segments ( segment_id TEXT PRIMARY KEY, stream TEXT NOT NULL, segment_index INTEGER NOT NULL, start_offset INTEGER NOT NULL, end_offset INTEGER NOT NULL, block_count INTEGER NOT NULL, last_append_ms INTEGER NOT NULL, size_bytes INTEGER NOT NULL, local_path TEXT NOT NULL, created_at_ms INTEGER NOT NULL, uploaded_at_ms INTEGER NULL, r2_etag TEXT NULL ); CREATE TABLE IF NOT EXISTS stream_segment_meta ( stream TEXT PRIMARY KEY, segment_count INTEGER NOT NULL, segment_offsets BLOB NOT NULL, segment_blocks BLOB NOT NULL, segment_last_ts BLOB NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS segments_stream_index_uniq ON segments(stream, segment_index); CREATE INDEX IF NOT EXISTS segments_stream_start_idx ON segments(stream, start_offset); CREATE INDEX IF NOT EXISTS segments_pending_upload_idx ON segments(uploaded_at_ms); CREATE TABLE IF NOT EXISTS manifests ( stream TEXT PRIMARY KEY, generation INTEGER NOT NULL, uploaded_generation INTEGER NOT NULL, last_uploaded_at_ms INTEGER NULL, last_uploaded_etag TEXT NULL ); CREATE TABLE IF NOT EXISTS schemas ( stream TEXT PRIMARY KEY, schema_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS producer_state ( stream TEXT NOT NULL, producer_id TEXT NOT NULL, epoch INTEGER NOT NULL, last_seq INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (stream, producer_id) ); CREATE TABLE IF NOT EXISTS stream_interpreters ( stream TEXT PRIMARY KEY, interpreted_through INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL ); -- Live dynamic template registry (per base stream). CREATE TABLE IF NOT EXISTS live_templates ( stream TEXT NOT NULL, template_id TEXT NOT NULL, entity TEXT NOT NULL, fields_json TEXT NOT NULL, encodings_json TEXT NOT NULL, state TEXT NOT NULL, created_at_ms INTEGER NOT NULL, last_seen_at_ms INTEGER NOT NULL, inactivity_ttl_ms INTEGER NOT NULL, active_from_source_offset INTEGER NOT NULL, retired_at_ms INTEGER NULL, retired_reason TEXT NULL, PRIMARY KEY (stream, template_id) ); CREATE INDEX IF NOT EXISTS live_templates_stream_entity_state_last_seen_idx ON live_templates(stream, entity, state, last_seen_at_ms); CREATE INDEX IF NOT EXISTS live_templates_stream_state_last_seen_idx ON live_templates(stream, state, last_seen_at_ms); `; var CREATE_INDEX_TABLES_SQL = ` CREATE TABLE IF NOT EXISTS index_state ( stream TEXT PRIMARY KEY, index_secret BLOB NOT NULL, indexed_through INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS index_runs ( run_id TEXT PRIMARY KEY, stream TEXT NOT NULL, level INTEGER NOT NULL, start_segment INTEGER NOT NULL, end_segment INTEGER NOT NULL, object_key TEXT NOT NULL, filter_len INTEGER NOT NULL, record_count INTEGER NOT NULL, retired_gen INTEGER NULL, retired_at_ms INTEGER NULL ); CREATE INDEX IF NOT EXISTS index_runs_stream_idx ON index_runs(stream, level, start_segment); `; var CREATE_TABLES_V4_SUFFIX_SQL = (suffix) => ` CREATE TABLE streams_${suffix} ( stream TEXT PRIMARY KEY, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, content_type TEXT NOT NULL, stream_seq TEXT NULL, closed INTEGER NOT NULL DEFAULT 0, closed_producer_id TEXT NULL, closed_producer_epoch INTEGER NULL, closed_producer_seq INTEGER NULL, ttl_seconds INTEGER NULL, epoch INTEGER NOT NULL, next_offset INTEGER NOT NULL, sealed_through INTEGER NOT NULL, uploaded_through INTEGER NOT NULL, uploaded_segment_count INTEGER NOT NULL DEFAULT 0, pending_rows INTEGER NOT NULL, pending_bytes INTEGER NOT NULL, last_append_ms INTEGER NOT NULL, last_segment_cut_ms INTEGER NOT NULL, segment_in_progress INTEGER NOT NULL, expires_at_ms INTEGER NULL, stream_flags INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE wal_${suffix} ( id INTEGER PRIMARY KEY, stream TEXT NOT NULL, offset INTEGER NOT NULL, ts_ms INTEGER NOT NULL, payload BLOB NOT NULL, payload_len INTEGER NOT NULL, routing_key BLOB NULL, content_type TEXT NULL, flags INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE segments_${suffix} ( segment_id TEXT PRIMARY KEY, stream TEXT NOT NULL, segment_index INTEGER NOT NULL, start_offset INTEGER NOT NULL, end_offset INTEGER NOT NULL, block_count INTEGER NOT NULL, last_append_ms INTEGER NOT NULL, size_bytes INTEGER NOT NULL, local_path TEXT NOT NULL, created_at_ms INTEGER NOT NULL, uploaded_at_ms INTEGER NULL, r2_etag TEXT NULL ); CREATE TABLE manifests_${suffix} ( stream TEXT PRIMARY KEY, generation INTEGER NOT NULL, uploaded_generation INTEGER NOT NULL, last_uploaded_at_ms INTEGER NULL, last_uploaded_etag TEXT NULL ); CREATE TABLE schemas_${suffix} ( stream TEXT PRIMARY KEY, schema_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL ); CREATE TABLE producer_state_${suffix} ( stream TEXT NOT NULL, producer_id TEXT NOT NULL, epoch INTEGER NOT NULL, last_seq INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (stream, producer_id) ); `; var CREATE_INDEXES_V4_SQL = ` CREATE UNIQUE INDEX IF NOT EXISTS wal_stream_offset_uniq ON wal(stream, offset); CREATE INDEX IF NOT EXISTS wal_stream_offset_idx ON wal(stream, offset); CREATE INDEX IF NOT EXISTS wal_ts_idx ON wal(ts_ms); CREATE INDEX IF NOT EXISTS streams_pending_bytes_idx ON streams(pending_bytes); CREATE INDEX IF NOT EXISTS streams_last_cut_idx ON streams(last_segment_cut_ms); CREATE INDEX IF NOT EXISTS streams_inprog_pending_idx ON streams(segment_in_progress, pending_bytes, last_segment_cut_ms); CREATE UNIQUE INDEX IF NOT EXISTS segments_stream_index_uniq ON segments(stream, segment_index); CREATE INDEX IF NOT EXISTS segments_stream_start_idx ON segments(stream, start_offset); CREATE INDEX IF NOT EXISTS segments_pending_upload_idx ON segments(uploaded_at_ms); `; function initSchema(db, opts = {}) { db.exec(DEFAULT_PRAGMAS_SQL); if (opts.skipMigrations) return; db.exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL);`); const readSchemaVersion = () => { const row = db.query("SELECT version FROM schema_version LIMIT 1;").get(); if (!row) return null; const raw = row.version; if (typeof raw === "bigint") return Number(raw); if (typeof raw === "number") return raw; return Number(raw); }; const version0 = readSchemaVersion(); if (version0 == null) { db.exec(CREATE_TABLES_V4_SQL); db.exec(CREATE_INDEX_TABLES_SQL); db.query("INSERT INTO schema_version(version) VALUES (?);").run(SCHEMA_VERSION); return; } if (version0 === SCHEMA_VERSION) return; let version = version0; while (version !== SCHEMA_VERSION) { if (version === 1) { migrateV1ToV4(db); } else if (version === 2) { migrateV2ToV4(db); } else if (version === 3) { migrateV3ToV4(db); } else if (version === 4) { migrateV4ToV5(db); } else if (version === 5) { migrateV5ToV6(db); } else if (version === 6) { migrateV6ToV7(db); } else if (version === 7) { migrateV7ToV8(db); } else if (version === 8) { migrateV8ToV9(db); } else if (version === 9) { migrateV9ToV10(db); } else if (version === 10) { migrateV10ToV11(db); } else { throw dsError(`unexpected schema version: ${version} (expected ${SCHEMA_VERSION})`); } const next = readSchemaVersion(); if (next == null) throw dsError("schema_version row missing after migration"); version = next; } } function migrateV1ToV4(db) { const tx = db.transaction(() => { db.exec(CREATE_TABLES_V4_SUFFIX_SQL("v4")); db.exec(` INSERT INTO streams_v4( stream, created_at_ms, updated_at_ms, content_type, stream_seq, closed, closed_producer_id, closed_producer_epoch, closed_producer_seq, ttl_seconds, epoch, next_offset, sealed_through, uploaded_through, pending_rows, pending_bytes, last_append_ms, last_segment_cut_ms, segment_in_progress, expires_at_ms, stream_flags ) SELECT stream, CAST(created_at_ns / 1000000 AS INTEGER), CAST(updated_at_ns / 1000000 AS INTEGER), 'application/octet-stream', NULL, 0, NULL, NULL, NULL, NULL, epoch, next_seq, sealed_through_seq, uploaded_through_seq, pending_rows, pending_bytes, CAST(last_append_ns / 1000000 AS INTEGER), CAST(last_segment_cut_ns / 1000000 AS INTEGER), segment_in_progress, CASE WHEN expires_at_ns IS NULL THEN NULL ELSE CAST(expires_at_ns / 1000000 AS INTEGER) END, CASE WHEN deleted != 0 THEN 1 ELSE 0 END FROM streams; `); db.exec(` INSERT INTO wal_v4( stream, offset, ts_ms, payload, payload_len, routing_key, content_type, flags ) SELECT stream, seq, CAST(append_ns / 1000000 AS INTEGER), payload, payload_len, CASE WHEN routing_key IS NULL THEN NULL ELSE CAST(routing_key AS BLOB) END, CASE WHEN is_json != 0 THEN 'application/json' ELSE NULL END, 0 FROM wal; `); db.exec(` INSERT INTO segments_v4( segment_id, stream, segment_index, start_offset, end_offset, block_count, last_append_ms, size_bytes, local_path, created_at_ms, uploaded_at_ms, r2_etag ) SELECT segment_id, stream, segment_index, start_seq, end_seq, block_count, CAST(last_append_ns / 1000000 AS INTEGER), size_bytes, local_path, CAST(created_at_ns / 1000000 AS INTEGER), CASE WHEN uploaded_at_ns IS NULL THEN NULL ELSE CAST(uploaded_at_ns / 1000000 AS INTEGER) END, NULL FROM segments; `); db.exec(` INSERT INTO manifests_v4( stream, generation, uploaded_generation, last_uploaded_at_ms, last_uploaded_etag ) SELECT stream, generation, uploaded_generation, CASE WHEN last_uploaded_at_ns IS NULL THEN NULL ELSE CAST(last_uploaded_at_ns / 1000000 AS INTEGER) END, last_uploaded_etag FROM manifests; `); db.exec(` INSERT INTO schemas_v4(stream, schema_json, updated_at_ms) SELECT stream, schema_json, CAST(updated_at_ns / 1000000 AS INTEGER) FROM schemas; `); db.exec(`DROP TABLE wal;`); db.exec(`DROP TABLE streams;`); db.exec(`DROP TABLE segments;`); db.exec(`DROP TABLE manifests;`); db.exec(`DROP TABLE schemas;`); db.exec(`ALTER TABLE streams_v4 RENAME TO streams;`); db.exec(`ALTER TABLE wal_v4 RENAME TO wal;`); db.exec(`ALTER TABLE segments_v4 RENAME TO segments;`); db.exec(`ALTER TABLE manifests_v4 RENAME TO manifests;`); db.exec(`ALTER TABLE schemas_v4 RENAME TO schemas;`); db.exec(`ALTER TABLE producer_state_v4 RENAME TO producer_state;`); db.exec(CREATE_INDEXES_V4_SQL); db.exec(CREATE_INDEX_TABLES_SQL); db.exec(`UPDATE schema_version SET version = 4;`); }); tx(); } function migrateV2ToV4(db) { const tx = db.transaction(() => { db.exec(`ALTER TABLE segments ADD COLUMN block_count INTEGER NOT NULL DEFAULT 0;`); db.exec(`ALTER TABLE segments ADD COLUMN last_append_ms INTEGER NOT NULL DEFAULT 0;`); db.exec(`ALTER TABLE streams ADD COLUMN content_type TEXT NOT NULL DEFAULT 'application/octet-stream';`); db.exec(`ALTER TABLE streams ADD COLUMN stream_seq TEXT NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN closed INTEGER NOT NULL DEFAULT 0;`); db.exec(`ALTER TABLE streams ADD COLUMN closed_producer_id TEXT NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN closed_producer_epoch INTEGER NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN closed_producer_seq INTEGER NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN ttl_seconds INTEGER NULL;`); db.exec(` CREATE TABLE IF NOT EXISTS producer_state ( stream TEXT NOT NULL, producer_id TEXT NOT NULL, epoch INTEGER NOT NULL, last_seq INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (stream, producer_id) ); `); db.exec(CREATE_INDEX_TABLES_SQL); db.exec(`UPDATE schema_version SET version = 4;`); }); tx(); } function migrateV3ToV4(db) { const tx = db.transaction(() => { db.exec(`ALTER TABLE streams ADD COLUMN content_type TEXT NOT NULL DEFAULT 'application/octet-stream';`); db.exec(`ALTER TABLE streams ADD COLUMN stream_seq TEXT NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN closed INTEGER NOT NULL DEFAULT 0;`); db.exec(`ALTER TABLE streams ADD COLUMN closed_producer_id TEXT NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN closed_producer_epoch INTEGER NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN closed_producer_seq INTEGER NULL;`); db.exec(`ALTER TABLE streams ADD COLUMN ttl_seconds INTEGER NULL;`); db.exec(` CREATE TABLE IF NOT EXISTS producer_state ( stream TEXT NOT NULL, producer_id TEXT NOT NULL, epoch INTEGER NOT NULL, last_seq INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (stream, producer_id) ); `); db.exec(CREATE_INDEX_TABLES_SQL); db.exec(`UPDATE schema_version SET version = 4;`); }); tx(); } function migrateV4ToV5(db) { const tx = db.transaction(() => { db.exec(CREATE_INDEX_TABLES_SQL); db.exec(`UPDATE schema_version SET version = 5;`); }); tx(); } function migrateV5ToV6(db) { const tx = db.transaction(() => { db.exec(`ALTER TABLE streams ADD COLUMN uploaded_segment_count INTEGER NOT NULL DEFAULT 0;`); db.exec(` CREATE TABLE IF NOT EXISTS stream_segment_meta ( stream TEXT PRIMARY KEY, segment_count INTEGER NOT NULL, segment_offsets BLOB NOT NULL, segment_blocks BLOB NOT NULL, segment_last_ts BLOB NOT NULL ); `); db.exec(`UPDATE schema_version SET version = 6;`); }); tx(); } function migrateV6ToV7(db) { const tx = db.transaction(() => { db.exec(` CREATE TABLE IF NOT EXISTS stream_interpreters ( stream TEXT PRIMARY KEY, interpreted_through INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL ); `); db.exec(`UPDATE schema_version SET version = 7;`); }); tx(); } function migrateV7ToV8(db) { const tx = db.transaction(() => { db.exec(`UPDATE schema_version SET version = 8;`); }); tx(); } function migrateV8ToV9(db) { const tx = db.transaction(() => { db.exec(` CREATE TABLE IF NOT EXISTS live_templates ( stream TEXT NOT NULL, template_id TEXT NOT NULL, entity TEXT NOT NULL, fields_json TEXT NOT NULL, encodings_json TEXT NOT NULL, state TEXT NOT NULL, created_at_ms INTEGER NOT NULL, last_seen_at_ms INTEGER NOT NULL, inactivity_ttl_ms INTEGER NOT NULL, active_from_source_offset INTEGER NOT NULL, retired_at_ms INTEGER NULL, retired_reason TEXT NULL, PRIMARY KEY (stream, template_id) ); `); db.exec(` CREATE INDEX IF NOT EXISTS live_templates_stream_entity_state_last_seen_idx ON live_templates(stream, entity, state, last_seen_at_ms); `); db.exec(` CREATE INDEX IF NOT EXISTS live_templates_stream_state_last_seen_idx ON live_templates(stream, state, last_seen_at_ms); `); db.exec(`UPDATE schema_version SET version = 9;`); }); tx(); } function migrateV9ToV10(db) { const tx = db.transaction(() => { db.exec(`ALTER TABLE streams ADD COLUMN wal_rows INTEGER NOT NULL DEFAULT 0;`); db.exec(`ALTER TABLE streams ADD COLUMN wal_bytes INTEGER NOT NULL DEFAULT 0;`); db.exec(`DROP TABLE IF EXISTS temp.wal_stats;`); db.exec(` CREATE TEMP TABLE wal_stats AS SELECT stream, COUNT(*) as rows, COALESCE(SUM(payload_len), 0) as bytes FROM wal GROUP BY stream; `); db.exec(` UPDATE streams SET wal_rows = COALESCE((SELECT rows FROM wal_stats WHERE wal_stats.stream = streams.stream), 0), wal_bytes = COALESCE((SELECT bytes FROM wal_stats WHERE wal_stats.stream = streams.stream), 0); `); db.exec(`DROP TABLE wal_stats;`); db.exec(`UPDATE schema_version SET version = 10;`); }); tx(); } function migrateV10ToV11(db) { const tx = db.transaction(() => { db.exec(`DROP INDEX IF EXISTS wal_touch_stream_rk_offset_idx;`); db.exec(`UPDATE schema_version SET version = ${SCHEMA_VERSION};`); }); tx(); } // src/sqlite/adapter.ts import { createRequire as createRequire2 } from "node:module"; // src/runtime/host_runtime.ts function detectHostRuntime() { return typeof globalThis.Bun !== "undefined" || Boolean(process.versions?.bun) ? "bun" : "node"; } // src/sqlite/adapter.ts class BunStatementAdapter { stmt; constructor(stmt) { this.stmt = stmt; } get(...params) { return this.stmt.get(...params); } all(...params) { return this.stmt.all(...params); } run(...params) { return this.stmt.run(...params); } iterate(...params) { return this.stmt.iterate(...params); } finalize() { if (typeof this.stmt.finalize === "function") this.stmt.finalize(); } } class BunDatabaseAdapter { db; constructor(db) { this.db = db; } exec(sql) { this.db.exec(sql); } query(sql) { return new BunStatementAdapter(this.db.query(sql)); } transaction(fn) { return this.db.transaction(fn); } close() { this.db.close(); } } class NodeStatementAdapter { stmt; constructor(stmt) { this.stmt = stmt; } get(...params) { return this.stmt.get(...params); } all(...params) { return this.stmt.all(...params); } run(...params) { return this.stmt.run(...params); } iterate(...params) { return this.stmt.iterate(...params); } finalize() { if (typeof this.stmt.finalize === "function") this.stmt.finalize(); } } class NodeDatabaseAdapter { txDepth = 0; txCounter = 0; db; constructor(db) { this.db = db; } exec(sql) { this.db.exec(sql); } query(sql) { const stmt = this.db.prepare(sql); if (typeof stmt?.setReadBigInts === "function") stmt.setReadBigInts(true); return new NodeStatementAdapter(stmt); } transaction(fn) { return () => { const nested = this.txDepth > 0; const savepoint = `ds_tx_${++this.txCounter}`; this.txDepth += 1; try { if (nested) this.db.exec(`SAVEPOINT ${savepoint};`); else this.db.exec("BEGIN;"); const out = fn(); if (nested) this.db.exec(`RELEASE SAVEPOINT ${savepoint};`); else this.db.exec("COMMIT;"); return out; } catch (err) { try { if (nested) { this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint};`); this.db.exec(`RELEASE SAVEPOINT ${savepoint};`); } else { this.db.exec("ROLLBACK;"); } } catch {} throw err; } finally { this.txDepth = Math.max(0, this.txDepth - 1); } }; } close() { this.db.close(); } } var openImpl = null; var openImplRuntime = null; var runtimeOverride = null; var require2 = createRequire2(import.meta.url); function selectedRuntime() { return runtimeOverride ?? detectHostRuntime(); } function buildOpenImpl(runtime) { if (runtime === "bun") { const { Database } = require2("bun:sqlite"); return (path) => new BunDatabaseAdapter(new Database(path)); } const { DatabaseSync } = require2("node:sqlite"); return (path) => new NodeDatabaseAdapter(new DatabaseSync(path)); } function openSqliteDatabase(path) { const runtime = selectedRuntime(); if (!openImpl || openImplRuntime !== runtime) { openImpl = buildOpenImpl(runtime); openImplRuntime = runtime; } if (!openImpl) throw dsError("sqlite adapter not initialized"); return openImpl(path); } // src/db/db.ts import { Result } from "better-result"; var STREAM_FLAG_DELETED = 1 << 0; var STREAM_FLAG_TOUCH = 1 << 1; var BASE_WAL_GC_CHUNK_OFFSETS = (() => { const raw = process.env.DS_BASE_WAL_GC_CHUNK_OFFSETS; if (raw == null || raw.trim() === "") return 1e5; const n = Number(raw); if (!Number.isFinite(n) || n <= 0) return 1e5; return Math.floor(n); })(); class SqliteDurableStore { db; dbstatReady = null; stmts; constructor(path, opts = {}) { this.db = openSqliteDatabase(path); initSchema(this.db, { skipMigrations: opts.skipMigrations }); if (opts.cacheBytes && opts.cacheBytes > 0) { const kb = Math.max(1, Math.floor(opts.cacheBytes / 1024)); this.db.exec(`PRAGMA cache_size = -${kb};`); } this.stmts = { getStream: this.db.query(`SELECT stream, created_at_ms, updated_at_ms, content_type, stream_seq, closed, closed_producer_id, closed_producer_epoch, closed_producer_seq, ttl_seconds, epoch, next_offset, sealed_through, uploaded_through, uploaded_segment_count, pending_rows, pending_bytes, wal_rows, wal_bytes, last_append_ms, last_segment_cut_ms, segment_in_progress, expires_at_ms, stream_flags FROM streams WHERE stream = ? LIMIT 1;`), upsertStream: this.db.query(`INSERT INTO streams(stream, created_at_ms, updated_at_ms, content_type, stream_seq, closed, closed_producer_id, closed_producer_epoch, closed_producer_seq, ttl_seconds, epoch, next_offset, sealed_through, uploaded_through, uploaded_segment_count, pending_rows, pending_bytes, wal_rows, wal_bytes, last_append_ms, last_segment_cut_ms, segment_in_progress, expires_at_ms, stream_flags) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET updated_at_ms=excluded.updated_at_ms, expires_at_ms=excluded.expires_at_ms, ttl_seconds=excluded.ttl_seconds, content_type=excluded.content_type, stream_flags=excluded.stream_flags;`), listStreams: this.db.query(`SELECT stream, created_at_ms, updated_at_ms, content_type, stream_seq, closed, closed_producer_id, closed_producer_epoch, closed_producer_seq, ttl_seconds, epoch, next_offset, sealed_through, uploaded_through, uploaded_segment_count, pending_rows, pending_bytes, wal_rows, wal_bytes, last_append_ms, last_segment_cut_ms, segment_in_progress, expires_at_ms, stream_flags FROM streams WHERE (stream_flags & ?) = 0 AND (expires_at_ms IS NULL OR expires_at_ms > ?) ORDER BY stream LIMIT ? OFFSET ?;`), setDeleted: this.db.query(`UPDATE streams SET stream_flags = (stream_flags | ?), updated_at_ms=? WHERE stream=?;`), insertWal: this.db.query(`INSERT INTO wal(stream, offset, ts_ms, payload, payload_len, routing_key, content_type, flags) VALUES(?, ?, ?, ?, ?, ?, ?, ?);`), updateStreamAppend: this.db.query(`UPDATE streams SET next_offset = ?, updated_at_ms = ?, last_append_ms = ?, pending_rows = pending_rows + ?, pending_bytes = pending_bytes + ?, wal_rows = wal_rows + ?, wal_bytes = wal_bytes + ? WHERE stream = ? AND (stream_flags & ?) = 0;`), updateStreamAppendSeqCheck: this.db.query(`UPDATE streams SET next_offset = ?, updated_at_ms = ?, last_append_ms = ?, pending_rows = pending_rows + ?, pending_bytes = pending_bytes + ?, wal_rows = wal_rows + ?, wal_bytes = wal_bytes + ? WHERE stream = ? AND (stream_flags & ?) = 0 AND next_offset = ?;`), candidateStreams: this.db.query(`SELECT stream, pending_bytes, pending_rows, last_segment_cut_ms, sealed_through, next_offset, epoch FROM streams WHERE (stream_flags & ?) = 0 AND segment_in_progress = 0 AND (pending_bytes >= ? OR pending_rows >= ? OR (? - last_segment_cut_ms) >= ?) ORDER BY pending_bytes DESC LIMIT ?;`), candidateStreamsNoInterval: this.db.query(`SELECT stream, pending_bytes, pending_rows, last_segment_cut_ms, sealed_through, next_offset, epoch FROM streams WHERE (stream_flags & ?) = 0 AND segment_in_progress = 0 AND (pending_bytes >= ? OR pending_rows >= ?) ORDER BY pending_bytes DESC LIMIT ?;`), listExpiredStreams: this.db.query(`SELECT stream FROM streams WHERE (stream_flags & ?) = 0 AND expires_at_ms IS NOT NULL AND expires_at_ms <= ? ORDER BY expires_at_ms ASC LIMIT ?;`), streamWalRange: this.db.query(`SELECT offset, ts_ms, routing_key, content_type, payload FROM wal WHERE stream = ? AND offset >= ? AND offset <= ? ORDER BY offset ASC;`), streamWalRangeByKey: this.db.query(`SELECT offset, ts_ms, routing_key, content_type, payload FROM wal WHERE stream = ? AND offset >= ? AND offset <= ? AND routing_key = ? ORDER BY offset ASC;`), createSegment: this.db.query(`INSERT INTO segments(segment_id, stream, segment_index, start_offset, end_offset, block_count, last_append_ms, size_bytes, local_path, created_at_ms, uploaded_at_ms, r2_etag) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL);`), listSegmentsForStream: this.db.query(`SELECT segment_id, stream, segment_index, start_offset, end_offset, block_count, last_append_ms, size_bytes, local_path, created_at_ms, uploaded_at_ms, r2_etag FROM segments WHERE stream=? ORDER BY segment_index ASC;`), getSegmentByIndex: this.db.query(`SELECT segment_id, stream, segment_index, start_offset, end_offset, block_count, last_append_ms, size_bytes, local_path, created_at_ms, uploaded_at_ms, r2_etag FROM segments WHERE stream=? AND segment_index=? LIMIT 1;`), findSegmentForOffset: this.db.query(`SELECT segment_id, stream, segment_index, start_offset, end_offset, block_count, last_append_ms, size_bytes, local_path, created_at_ms, uploaded_at_ms, r2_etag FROM segments WHERE stream=? AND start_offset <= ? AND end_offset >= ? ORDER BY segment_index DESC LIMIT 1;`), nextSegmentIndex: this.db.query(`SELECT COALESCE(MAX(segment_index)+1, 0) as next_idx FROM segments WHERE stream=?;`), markSegmentUploaded: this.db.query(`UPDATE segments SET r2_etag=?, uploaded_at_ms=? WHERE segment_id=?;`), pendingUploadSegments: this.db.query(`SELECT segment_id, stream, segment_index, start_offset, end_offset, block_count, last_append_ms, size_bytes, local_path, created_at_ms, uploaded_at_ms, r2_etag FROM segments WHERE uploaded_at_ms IS NULL ORDER BY created_at_ms ASC LIMIT ?;`), countPendingSegments: this.db.query(`SELECT COUNT(*) as cnt FROM segments WHERE uploaded_at_ms IS NULL;`), countSegmentsForStream: this.db.query(`SELECT COUNT(*) as cnt FROM segments WHERE stream=?;`), tryClaimSegment: this.db.query(`UPDATE streams SET segment_in_progress=1, updated_at_ms=? WHERE stream=? AND segment_in_progress=0;`), getManifest: this.db.query(`SELECT stream, generation, uploaded_generation, last_uploaded_at_ms, last_uploaded_etag FROM manifests WHERE stream=? LIMIT 1;`), upsertManifest: this.db.query(`INSERT INTO manifests(stream, generation, uploaded_generation, last_uploaded_at_ms, last_uploaded_etag) VALUES(?, ?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET generation=excluded.generation, uploaded_generation=excluded.uploaded_generation, last_uploaded_at_ms=excluded.last_uploaded_at_ms, last_uploaded_etag=excluded.last_uploaded_etag;`), getIndexState: this.db.query(`SELECT stream, index_secret, indexed_through, updated_at_ms FROM index_state WHERE stream=? LIMIT 1;`), upsertIndexState: this.db.query(`INSERT INTO index_state(stream, index_secret, indexed_through, updated_at_ms) VALUES(?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET index_secret=excluded.index_secret, indexed_through=excluded.indexed_through, updated_at_ms=excluded.updated_at_ms;`), updateIndexedThrough: this.db.query(`UPDATE index_state SET indexed_through=?, updated_at_ms=? WHERE stream=?;`), listIndexRuns: this.db.query(`SELECT run_id, stream, level, start_segment, end_segment, object_key, filter_len, record_count, retired_gen, retired_at_ms FROM index_runs WHERE stream=? AND retired_gen IS NULL ORDER BY start_segment ASC, level ASC;`), listIndexRunsAll: this.db.query(`SELECT run_id, stream, level, start_segment, end_segment, object_key, filter_len, record_count, retired_gen, retired_at_ms FROM index_runs WHERE stream=? ORDER BY start_segment ASC, level ASC;`), listRetiredIndexRuns: this.db.query(`SELECT run_id, stream, level, start_segment, end_segment, object_key, filter_len, record_count, retired_gen, retired_at_ms FROM index_runs WHERE stream=? AND retired_gen IS NOT NULL ORDER BY retired_at_ms ASC;`), insertIndexRun: this.db.query(`INSERT OR IGNORE INTO index_runs(run_id, stream, level, start_segment, end_segment, object_key, filter_len, record_count, retired_gen, retired_at_ms) VALUES(?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL);`), retireIndexRun: this.db.query(`UPDATE index_runs SET retired_gen=?, retired_at_ms=? WHERE run_id=?;`), deleteIndexRun: this.db.query(`DELETE FROM index_runs WHERE run_id=?;`), countUploadedSegments: this.db.query(`SELECT COALESCE(MAX(segment_index), -1) as max_idx FROM segments WHERE stream=? AND r2_etag IS NOT NULL;`), getSegmentMeta: this.db.query(`SELECT stream, segment_count, segment_offsets, segment_blocks, segment_last_ts FROM stream_segment_meta WHERE stream=? LIMIT 1;`), ensureSegmentMeta: this.db.query(`INSERT INTO stream_segment_meta(stream, segment_count, segment_offsets, segment_blocks, segment_last_ts) VALUES(?, 0, x'', x'', x'') ON CONFLICT(stream) DO NOTHING;`), appendSegmentMeta: this.db.query(`UPDATE stream_segment_meta SET segment_count = segment_count + 1, segment_offsets = segment_offsets || ?, segment_blocks = segment_blocks || ?, segment_last_ts = segment_last_ts || ? WHERE stream = ?;`), upsertSegmentMeta: this.db.query(`INSERT INTO stream_segment_meta(stream, segment_count, segment_offsets, segment_blocks, segment_last_ts) VALUES(?, ?, ?, ?, ?) ON CONFLICT(stream) DO UPDATE SET segment_count=excluded.segment_count, segment_offsets=excluded.segment_offsets, segment_blocks=excluded.segment_blocks, segment_last_ts=excluded.segment_last_ts;`), setUploadedSegmentCount: this.db.query(`UPDATE streams SET uploaded_segment_count=?, updated_at_ms=? WHERE stream=?;`), advanceUploadedThrough: this.db.query(`UPDATE streams SET uploaded_through=?, updated_at_ms=? WHERE stream=?;`), deleteWalBeforeOffset: this.db.query(`DELETE FROM wal WHERE stream=? AND offset <= ?;`), getSchemaRegistry: this.db.query(`SELECT stream, schema_json, updated_at_ms FROM schemas WHERE stream=? LIMIT 1;`), upsertSchemaRegistry: this.db.query(`INSERT INTO schemas(stream, schema_json, updated_at_ms) VALUES(?, ?, ?) ON CONFLICT(stream) DO UPDATE SET schema_json=excluded.schema_json, updated_at_ms=excluded.updated_at_ms;`), getStreamInterpreter: this.db.query(`SELECT stream, interpreted_through, updated_at_ms FROM stream_interpreters WHERE stream=? LIMIT 1;`), upsertStreamInterpreter: this.db.query(`INSERT INTO stream_interpreters(stream, interpreted_through, updated_at_ms) VALUES(?, ?, ?) ON CONFLICT(stream) DO UPDATE SET interpreted_through=excluded.interpreted_through, updated_at_ms=excluded.updated_at_ms;`), deleteStreamInterpreter: this.db.query(`DELETE FROM stream_interpreters WHERE stream=?;`), listStreamInterpreters: this.db.query(`SELECT stream, interpreted_through, updated_at_ms FROM stream_interpreters ORDER BY stream ASC;`), countStreams: this.db.query(`SELECT COUNT(*) as cnt FROM streams WHERE (stream_flags & ?) = 0;`), sumPendingBytes: this.db.query(`SELECT COALESCE(SUM(pending_bytes), 0) as total FROM streams;`), sumPendingSegmentBytes: this.db.query(`SELECT COALESCE(SUM(size_bytes), 0) as total FROM segments WHERE uploaded_at_ms IS NULL;`) }; } toBigInt(v) { return typeof v === "bigint" ? v : BigInt(v); } bindInt(v) { const max = BigInt(Number.MAX_SAFE_INTEGER); const min = BigInt(Number.MIN_SAFE_INTEGER); if (v <= max && v >= min) return Number(v); return v.toString(); } encodeU64Le(value) { const buf = new Uint8Array(8); const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); dv.setBigUint64(0, value, true); return buf; } encodeU32Le(value) { const buf = new Uint8Array(4); const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); dv.setUint32(0, value >>> 0, true); return buf; } coerceStreamRow(row) { return { stream: String(row.stream), created_at_ms: this.toBigInt(row.created_at_ms), updated_at_ms: this.toBigInt(row.updated_at_ms), content_type: String(row.content_type), stream_seq: row.stream_seq == null ? null : String(row.stream_seq), closed: Number(row.closed), closed_producer_id: row.closed_producer_id == null ? null : String(row.closed_producer_id), closed_producer_epoch: row.closed_producer_epoch == null ? null : Number(row.closed_producer_epoch), closed_producer_seq: row.closed_producer_seq == null ? null : Number(row.closed_producer_seq), ttl_seconds: row.ttl_seconds == null ? null : Number(row.ttl_seconds), epoch: Number(row.epoch), next_offset: this.toBigInt(row.next_offset), sealed_through: this.toBigInt(row.sealed_through), uploaded_through: this.toBigInt(row.uploaded_through), uploaded_segment_count: Number(row.uploaded_segment_count ?? 0), pending_rows: this.toBigInt(row.pending_rows), pending_bytes: this.toBigInt(row.pending_bytes), wal_rows: this.toBigInt(row.wal_rows ?? 0), wal_bytes: this.toBigInt(row.wal_bytes ?? 0), last_append_ms: this.toBigInt(row.last_append_ms), last_segment_cut_ms: this.toBigInt(row.last_segment_cut_ms), segment_in_progress: Number(row.segment_in_progress), expires_at_ms: row.expires_at_ms == null ? null : this.toBigInt(row.expires_at_ms), stream_flags: Number(row.stream_flags) }; } coerceSegmentRow(row) { return { segment_id: String(row.segment_id), stream: String(row.stream), segment_index: Number(row.segment_index), start_offset: this.toBigInt(row.start_offset), end_offset: this.toBigInt(row.end_offset), block_count: Number(row.block_count), last_append_ms: this.toBigInt(row.last_append_ms), size_bytes: Number(row.size_bytes), local_path: String(row.local_path), created_at_ms: this.toBigInt(row.created_at_ms), uploaded_at_ms: row.uploaded_at_ms == null ? null : this.toBigInt(row.uploaded_at_ms), r2_etag: row.r2_etag == null ? null : String(row.r2_etag) }; } close() { this.db.close(); } nowMs() { return BigInt(Date.now()); } isDeleted(row) { return (row.stream_flags & STREAM_FLAG_DELETED) !== 0; } getStream(stream) { const row = this.stmts.getStream.get(stream); return row ? this.coerceStreamRow(row) : null; } ensureStream(stream, opts) { const existing = this.getStream(stream); if (existing) return existing; const now = this.nowMs(); const epoch = 0; const nextOffset = 0n; const contentType = opts?.contentType ?? "application/octet-stream"; const closed = opts?.closed ? 1 : 0; const closedProducer = opts?.closedProducer ?? null; const expiresAtMs = opts?.expiresAtMs ?? null; const ttlSeconds = opts?.ttlSeconds ?? null; const streamFlags = opts?.streamFlags ?? 0; this.db.query(`INSERT INTO streams( stream, created_at_ms, updated_at_ms, content_type, stream_seq, closed, closed_producer_id, closed_producer_epoch, closed_producer_seq, ttl_seconds, epoch, next_offset, sealed_through, uploaded_through, uploaded_segment_count, pending_rows, pending_bytes, last_append_ms, last_segment_cut_ms, segment_in_progress, expires_at_ms, stream_flags ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`).run(stream, now, now, contentType, null, closed, closedProducer ? closedProducer.id : null, closedProducer ? closedProducer.epoch : null, closedProducer ? closedProducer.seq : null, ttlSeconds, epoch, nextOffset, -1n, -1n, 0, 0n, 0n, now, now, 0, expiresAtMs, streamFlags); this.stmts.upsertManifest.run(stream, 0, 0, null, null); this.ensureSegmentMeta(stream); return this.getStream(stream); } restoreStreamRow(row) { this.stmts.upsertStream.run(row.stream, row.created_at_ms, row.updated_at_ms, row.content_type, row.stream_seq, row.closed, row.closed_producer_id, row.closed_producer_epoch, row.closed_producer_seq, row.ttl_seconds, row.epoch, row.next_offset, row.sealed_through, row.uploaded_through, row.uploaded_segment_count, row.pending_rows, row.pending_bytes, row.wal_rows, row.wal_bytes, row.last_append_ms, row.last_segment_cut_ms, row.segment_in_progress, row.expires_at_ms, row.stream_flags); } listStreams(limit, offset) { const now = this.nowMs(); const rows = this.stmts.listStreams.all(STREAM_FLAG_DELETED | STREAM_FLAG_TOUCH, now, limit, offset); return rows.map((r) => this.coerceStreamRow(r)); } listExpiredStreams(limit) { const now = this.nowMs(); const rows = this.stmts.listExpiredStreams.all(STREAM_FLAG_DELETED | STREAM_FLAG_TOUCH, now, limit); return rows.map((r) => String(r.stream)); } deleteStream(stream) { const existing = this.getStream(stream); if (!existing) return false; const now = this.nowMs(); this.stmts.setDeleted.run(STREAM_FLAG_DELETED, now, stream); return true; } hardDeleteStream(stream) { const tx = this.db.transaction(() => { const existing = this.getStream(stream); if (!existing) return false; this.db.query(`DELETE FROM wal WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM segments WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM manifests WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM schemas WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM stream_interpreters WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM live_templates WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM producer_state WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM index_state WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM index_runs WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM stream_segment_meta WHERE stream=?;`).run(stream); this.db.query(`DELETE FROM streams WHERE stream=?;`).run(stream); return true; }); return tx(); } getSchemaRegistry(stream) { const row = this.stmts.getSchemaRegistry.get(stream); if (!row) return null; return { stream: String(row.stream), registry_json: String(row.schema_json), updated_at_ms: this.toBigInt(row.updated_at_ms) }; } upsertSchemaRegistry(stream, registryJson) { this.stmts.upsertSchemaRegistry.run(stream, registryJson, this.nowMs()); } getStreamInterpreter(stream) { const row = this.stmts.getStreamInterpreter.get(stream); if (!row) return null; return { stream: String(row.stream), interpreted_through: this.toBigInt(row.interpreted_through), updated_at_ms: this.toBigInt(row.updated_at_ms) }; } listStreamInterpreters() { const rows = this.stmts.listStreamInterpreters.all(); return rows.map((row) => ({ stream: String(row.stream), interpreted_through: this.toBigInt(row.interpreted_through), updated_at_ms: this.toBigInt(row.updated_at_ms) })); } ensureStreamInterpreter(stream) { const existing = this.getStreamInterpreter(stream); if (existing) return; const srow = this.getStream(stream); const initialThrough = srow ? srow.next_offset - 1n : -1n; this.stmts.upsertStreamInterpreter.run(stream, this.bindInt(initialThrough), this.nowMs()); } updateStreamInterpreterThrough(stream, interpretedThrough) { this.stmts.upsertStreamInterpreter.run(stream, this.bindInt(interpretedThrough), this.nowMs()); } deleteStreamInterpreter(stream) { this.stmts.deleteStreamInterpreter.run(stream); } addStreamFlags(stream, flags) { if (!Number.isFinite(flags) || flags <= 0) return; this.db.query(`UPDATE streams SET stream_flags = (stream_flags | ?), updated_at_ms=? WHERE stream=?;`).run(flags, this.nowMs(), stream); } getWalOldestOffset(stream) { const row = this.db.query(`SELECT MIN(offset) as min_off FROM wal WHERE stream=?;`).get(stream); if (!row || row.min_off == null) return null; return this.toBigInt(row.min_off); } trimWalByAge(stream, maxAgeMs) { const ageMs = Math.max(0, Math.floor(maxAgeMs)); if (!Number.isFinite(ageMs)) return { trimmedRows: 0, trimmedBytes: 0, keptFromOffset: null }; const tx = this.db.transaction(() => { const lastRow = this.db.query(`SELECT offset, ts_ms FROM wal WHERE stream=? ORDER BY offset DESC LIMIT 1;`).get(stream); if (!lastRow || lastRow.offset == null) return { trimmedRows: 0, trimmedBytes: 0, keptFromOffset: null }; const lastOffset = this.toBigInt(lastRow.offset); let keepFromOffset; if (ageMs === 0) { keepFromOffset = lastOffset; } else { const cutoff = this.nowMs() - BigInt(ageMs); const keepRow = this.db.query(`SELECT offset FROM wal WHERE stream=? AND ts_ms >= ? ORDER BY offset ASC LIMIT 1;`).get(stream, this.bindInt(cutoff)); keepFromOffset = keepRow && keepRow.offset != null ? this.toBigInt(keepRow.offset) : lastOffset; } if (keepFromOffset <= 0n) return { trimmedRows: 0, trimmedBytes: 0, keptFromOffset: keepFromOffset }; const stats = this.db.query(`SELECT COALESCE(SUM(payload_len), 0) as bytes, COUNT(*) as rows FROM wal WHERE stream=? AND offset < ?;`).get(stream, this.bindInt(keepFromOffset)); const bytes = this.toBigInt(stats?.bytes ?? 0); const rows = this.toBigInt(stats?.rows ?? 0); if (rows <= 0n) return { trimmedRows: 0, trimmedBytes: 0, keptFromOffset: keepFromOffset }; this.db.query(`DELETE FROM wal WHERE stream=? AND offset < ?;`).run(stream, this.bindInt(keepFromOffset)); const now = this.nowMs(); this.db.query(`UPDATE streams SET pending_bytes = CASE WHEN pending_bytes >= ? THEN pending_bytes - ? ELSE 0 END, pending_rows = CASE WHEN pending_rows >= ? THEN pending_rows - ? ELSE 0 END, wal_bytes = CASE WHEN wal_bytes >= ? THEN wal_bytes - ? ELSE 0 END, wal_rows = CASE WHEN wal_rows >= ? THEN wal_rows - ? ELSE 0 END, updated_at_ms = ? WHERE stream = ?;`).run(bytes, bytes, rows, rows, bytes, bytes, rows, rows, now, stream); const trimmedBytes = bytes <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(bytes) : Number.MAX_SAFE_INTEGER; const trimmedRows = rows <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(rows) : Number.MAX_SAFE_INTEGER; return { trimmedRows, trimmedBytes, keptFromOffset: keepFromOffset }; }); return tx(); } countStreams() { const row = this.stmts.countStreams.get(STREAM_FLAG_DELETED | STREAM_FLAG_TOUCH); return row ? Number(row.cnt) : 0; } sumPendingBytes() { const row = this.stmts.sumPendingBytes.get(); const total = row?.total ?? 0; return Number(this.toBigInt(total)); } sumPendingSegmentBytes() { const row = this.stmts.sumPendingSegmentBytes.get(); const total = row?.total ?? 0; return Number(this.toBigInt(total)); } ensureDbStat() { if (this.dbstatReady != null) return this.dbstatReady; try { this.db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS temp.dbstat USING dbstat;"); this.dbstatReady = true; } catch { this.dbstatReady = false; } return this.dbstatReady; } estimateWalBytes() { try { const row = this.db.query(`SELECT COALESCE(SUM(payload_len), 0) as payload, COALESCE(SUM(LENGTH(routing_key)), 0) as rk, COALESCE(SUM(LENGTH(content_type)), 0) as ct FROM wal;`).get(); return Number(row?.payload ?? 0) + Number(row?.rk ?? 0) + Number(row?.ct ?? 0); } catch { return 0; } } estimateMetaBytes() { try { const streams = this.db.query(`SELECT COALESCE(SUM(LENGTH(stream)), 0) as stream, COALESCE(SUM(LENGTH(content_type)), 0) as content_type, COALESCE(SUM(LENGTH(stream_seq)), 0) as stream_seq, COALESCE(SUM(LENGTH(closed_producer_id)), 0) as closed_producer_id FROM streams;`).get(); const segments = this.db.query(`SELECT COALESCE(SUM(LENGTH(segment_id)), 0) as segment_id, COALESCE(SUM(LENGTH(stream)), 0) as stream, COALESCE(SUM(LENGTH(local_path)), 0) as local_path, COALESCE(SUM(LENGTH(r2_etag)), 0) as r2_etag FROM segments;`).get(); const manifests = this.db.query(`SELECT COALESCE(SUM(LENGTH(stream)), 0) as stream, COALESCE(SUM(LENGTH(last_uploaded_etag)), 0) as last_uploaded_etag FROM manifests;`).get(); const schemas = this.db.query(`SELECT COALESCE(SUM(LENGTH(schema_json)), 0) as schema_json FROM schemas;`).get(); const producers = this.db.query(`SELECT COALESCE(SUM(LENGTH(stream)), 0) as stream, COALESCE(SUM(LENGTH(producer_id)), 0) as producer_id FROM producer_state;`).get(); const total = Number(streams?.stream ?? 0) + Number(streams?.content_type ?? 0) + Number(streams?.stream_seq ?? 0) + Number(streams?.closed_producer_id ?? 0) + Number(segments?.segment_id ?? 0) + Number(segments?.stream ?? 0) + Number(segments?.local_path ?? 0) + Number(segments?.r2_etag ?? 0) + Number(manifests?.stream ?? 0) + Number(manifests?.last_uploaded_etag ?? 0) + Number(schemas?.schema_json ?? 0) + Number(producers?.stream ?? 0) + Number(producers?.producer_id ?? 0); return total; } catch { return 0; } } getWalDbSizeBytes() { if (this.ensureDbStat()) { try { const row = this.db.query(`SELECT COALESCE(SUM(pgsize), 0) as total FROM temp.dbstat WHERE name = 'wal';`).get(); return Number(row?.total ?? 0); } catch {} } return this.estimateWalBytes(); } getMetaDbSizeBytes() { if (this.ensureDbStat()) { try { const row = this.db.query(`SELECT COALESCE(SUM(pgsize), 0) as total FROM temp.dbstat WHERE name != 'wal';`).get(); return Number(row?.total ?? 0); } catch {} } return this.estimateMetaBytes(); } appendWalRows(args) { const { stream, startOffset, expectedOffset, rows } = args; if (rows.length === 0) return Result.err({ kind: "no_rows" }); const tx = this.db.transaction(() => { const st = this.getStream(stream); if (!st || this.isDeleted(st)) return Result.err({ kind: "stream_missing" }); if (st.expires_at_ms != null && this.nowMs() > st.expires_at_ms) return Result.err({ kind: "stream_expired" }); if (expectedOffset !== undefined && st.next_offset !== expectedOffset) { return Result.err({ kind: "seq_mismatch", expectedNext: st.next_offset }); } let totalBytes = 0n; let offset = startOffset; for (const r of rows) { const payloadLen = r.payload.byteLength; totalBytes += BigInt(payloadLen); this.stmts.insertWal.run(stream, offset, r.appendMs, r.payload, payloadLen, r.routingKey, r.contentType, 0); offset += 1n; } const lastOffset = offset - 1n; const newNextOffset = lastOffset + 1n; const now = this.nowMs(); const pendingRows = BigInt(rows.length); const lastAppend = rows[rows.length - 1].appendMs; this.stmts.updateStreamAppend.run(newNextOffset, now, lastAppend, pendingRows, totalBytes, pendingRows, totalBytes, stream, STREAM_FLAG_DELETED); return Result.ok({ lastOffset }); }); return tx(); } *iterWalRange(stream, startOffset, endOffset, routingKey) { const start = this.bindInt(startOffset); const end = this.bindInt(endOffset); const stmt = routingKey ? this.db.query(`SELECT offset, ts_ms, routing_key, content_type, payload FROM wal WHERE stream = ? AND offset >= ? AND offset <= ? AND routing_key = ? ORDER BY offset ASC;`) : this.db.query(`SELECT offset, ts_ms, routing_key, content_type, payload FROM wal WHERE stream = ? AND offset >= ? AND offset <= ? ORDER BY offset ASC;`); try { const it = routingKey ? stmt.iterate(stream, start, end, routingKey) : stmt.iterate(stream, start, end); for (const row of it) { yield row; } } finally { try { stmt.finalize?.(); } catch {} } } nextSegmentIndexForStream(stream) { const row = this.stmts.nextSegmentIndex.get(stream); return Number(row?.next_idx ?? 0); } createSegmentRow(row) { this.stmts.createSegment.run(row.segmentId, row.stream, row.segmentIndex, row.startOffset, row.endOffset, row.blockCount, row.lastAppendMs, row.sizeBytes, row.localPath, this.nowMs()); } commitSealedSegment(row) { const tx = this.db.transaction(() => { this.createSegmentRow(row); this.appendSegmentMeta(row.stream, row.endOffset + 1n, row.blockCount, row.lastAppendMs * 1000000n); this.setStreamSealedThrough(row.stream, row.endOffset, row.payloadBytes, row.rowsSealed); }); tx(); } listSegmentsForStream(stream) { const rows = this.stmts.listSegmentsForStream.all(stream); return rows.map((r) => this.coerceSegmentRow(r)); } getSegmentByIndex(stream, segmentIndex) { const row = this.stmts.getSegmentByIndex.get(stream, segmentIndex); return row ? this.coerceSegmentRow(row) : null; } findSegmentForOffset(stream, offset) { const bound = this.bindInt(offset); const row = this.stmts.findSegmentForOffset.get(stream, bound, bound); return row ? this.coerceSegmentRow(row) : null; } pendingUploadSegments(limit) { const rows = this.stmts.pendingUploadSegments.all(limit); return rows.map((r) => this.coerceSegmentRow(r)); } countPendingSegments() { const row = this.stmts.countPendingSegments.get(); return row ? Number(row.cnt) : 0; } countSegmentsForStream(stream) { const row = this.stmts.countSegmentsForStream.get(stream); return row ? Number(row.cnt) : 0; } getSegmentMeta(stream) { const row = this.stmts.getSegmentMeta.get(stream); if (!row) return null; const offsets = row.segment_offsets instanceof Uint8Array ? row.segment_offsets : new Uint8Array(row.segment_offsets); const blocks = row.segment_blocks instanceof Uint8Array ? row.segment_blocks : new Uint8Array(row.segment_blocks); const lastTs = row.segment_last_ts instanceof Uint8Array ? row.segment_last_ts : new Uint8Array(row.segment_last_ts); return { stream: String(row.stream), segment_count: Number(row.segment_count), segment_offsets: offsets, segment_blocks: blocks, segment_last_ts: lastTs }; } ensureSegmentMeta(stream) { this.stmts.ensureSegmentMeta.run(stream); } appendSegmentMeta(stream, offsetPlusOne, blockCount, lastAppendNs) { this.ensureSegmentMeta(stream); const offsetBytes = this.encodeU64Le(offsetPlusOne); const blockBytes = this.encodeU32Le(blockCount); const tsBytes = this.encodeU64Le(lastAppendNs); this.stmts.appendSegmentMeta.run(offsetBytes, blockBytes, tsBytes, stream); } upsertSegmentMeta(stream, count, offsets, blocks, lastTs) { this.stmts.upsertSegmentMeta.run(stream, count, offsets, blocks, lastTs); } rebuildSegmentMeta(stream) { const rows = this.db.query(`SELECT end_offset, block_count, last_append_ms FROM segments WHERE stream=? ORDER BY segment_index ASC;`).all(stream); const count = rows.length; const offsets = new Uint8Array(count * 8); const blocks = new Uint8Array(count * 4); const lastTs = new Uint8Array(count * 8); const dvOffsets = new DataView(offsets.buffer, offsets.byteOffset, offsets.byteLength); const dvBlocks = new DataView(blocks.buffer, blocks.byteOffset, blocks.byteLength); const dvLastTs = new DataView(lastTs.buffer, lastTs.byteOffset, lastTs.byteLength); for (let i = 0;i < rows.length; i++) { const endOffset = this.toBigInt(rows[i].end_offset); const blockCount = Number(rows[i].block_count); const lastAppendMs = this.toBigInt(rows[i].last_append_ms); dvOffsets.setBigUint64(i * 8, endOffset + 1n, true); dvBlocks.setUint32(i * 4, blockCount >>> 0, true); dvLastTs.setBigUint64(i * 8, lastAppendMs * 1000000n, true); } this.upsertSegmentMeta(stream, count, offsets, blocks, lastTs); return { stream, segment_count: count, segment_offsets: offsets, segment_blocks: blocks, segment_last_ts: lastTs }; } setUploadedSegmentCount(stream, count) { this.stmts.setUploadedSegmentCount.run(count, this.nowMs(), stream); } advanceUploadedSegmentCount(stream) { const row = this.getStream(stream); if (!row) return 0; let count = row.uploaded_segment_count ?? 0; for (;; ) { const seg = this.getSegmentByIndex(stream, count); if (!seg || !seg.r2_etag) break; count += 1; } if (count !== row.uploaded_segment_count) { this.stmts.setUploadedSegmentCount.run(count, this.nowMs(), stream); } return count; } markSegmentUploaded(segmentId, etag, uploadedAtMs) { this.stmts.markSegmentUploaded.run(etag, uploadedAtMs, segmentId); } setStreamSealedThrough(stream, sealedThrough, bytesSealed, rowsSealed) { const now = this.nowMs(); this.db.query(`UPDATE streams SET sealed_through = ?, pending_bytes = CASE WHEN pending_bytes >= ? THEN pending_bytes - ? ELSE 0 END, pending_rows = CASE WHEN pending_rows >= ? THEN pending_rows - ? ELSE 0 END, last_segment_cut_ms = ?, updated_at_ms = ? WHERE stream = ?;`).run(sealedThrough, bytesSealed, bytesSealed, rowsSealed, rowsSealed, now, now, stream); } setSegmentInProgress(stream, inProgress) { this.db.query(`UPDATE streams SET segment_in_progress=?, updated_at_ms=? WHERE stream=?;`).run(inProgress, this.nowMs(), stream); } tryClaimSegment(stream) { const res = this.stmts.tryClaimSegment.run(this.nowMs(), stream); const changes = typeof res?.changes === "bigint" ? res.changes : BigInt(Number(res?.changes ?? 0)); return changes > 0n; } resetSegmentInProgress() { this.db.query(`UPDATE streams SET segment_in_progress=0 WHERE segment_in_progress != 0;`).run(); } advanceUploadedThrough(stream, uploadedThrough) { this.stmts.advanceUploadedThrough.run(uploadedThrough, this.nowMs(), stream); } deleteWalThrough(stream, uploadedThrough) { const through = this.bindInt(uploadedThrough); const tx = this.db.transaction(() => { const stats = this.db.query(`SELECT COALESCE(SUM(payload_len), 0) as bytes, COUNT(*) as rows FROM wal WHERE stream=? AND offset <= ?;`).get(stream, through); const bytes = this.toBigInt(stats?.bytes ?? 0); const rows = this.toBigInt(stats?.rows ?? 0); if (rows <= 0n) return { deletedRows: 0, deletedBytes: 0 }; this.stmts.deleteWalBeforeOffset.run(stream, through); const now = this.nowMs(); this.db.query(`UPDATE streams SET wal_bytes = CASE WHEN wal_bytes >= ? THEN wal_bytes - ? ELSE 0 END, wal_rows = CASE WHEN wal_rows >= ? THEN wal_rows - ? ELSE 0 END, updated_at_ms = ? WHERE stream = ?;`).run(bytes, bytes, rows, rows, now, stream); const deletedBytes = bytes <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(bytes) : Number.MAX_SAFE_INTEGER; const deletedRows = rows <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(rows) : Number.MAX_SAFE_INTEGER; return { deletedRows, deletedBytes }; }); return tx(); } getManifestRow(stream) { const row = this.stmts.getManifest.get(stream); if (!row) { this.stmts.upsertManifest.run(stream, 0, 0, null, null); const fresh = this.stmts.getManifest.get(stream); return { stream: String(fresh.stream), generation: Number(fresh.generation), uploaded_generation: Number(fresh.uploaded_generation), last_uploaded_at_ms: fresh.last_uploaded_at_ms == null ? null : this.toBigInt(fresh.last_uploaded_at_ms), last_uploaded_etag: fresh.last_uploaded_etag == null ? null : String(fresh.last_uploaded_etag) }; } return { stream: String(row.stream), generation: Number(row.generation), uploaded_generation: Number(row.uploaded_generation), last_uploaded_at_ms: row.last_uploaded_at_ms == null ? null : this.toBigInt(row.last_uploaded_at_ms), last_uploaded_etag: row.last_uploaded_etag == null ? null : String(row.last_uploaded_etag) }; } upsertManifestRow(stream, generation, uploadedGeneration, uploadedAtMs, etag) { this.stmts.upsertManifest.run(stream, generation, uploadedGeneration, uploadedAtMs, etag); } getIndexState(stream) { const row = this.stmts.getIndexState.get(stream); if (!row) return null; return { stream: String(row.stream), index_secret: row.index_secret instanceof Uint8Array ? row.index_secret : new Uint8Array(row.index_secret), indexed_through: Number(row.indexed_through), updated_at_ms: this.toBigInt(row.updated_at_ms) }; } upsertIndexState(stream, indexSecret, indexedThrough) { this.stmts.upsertIndexState.run(stream, indexSecret, indexedThrough, this.nowMs()); } updateIndexedThrough(stream, indexedThrough) { this.stmts.updateIndexedThrough.run(indexedThrough, this.nowMs(), stream); } listIndexRuns(stream) { const rows = this.stmts.listIndexRuns.all(stream); return rows.map((r) => ({ run_id: String(r.run_id), stream: String(r.stream), level: Number(r.level), start_segment: Number(r.start_segment), end_segment: Number(r.end_segment), object_key: String(r.object_key), filter_len: Number(r.filter_len), record_count: Number(r.record_count), retired_gen: r.retired_gen == null ? null : Number(r.retired_gen), retired_at_ms: r.retired_at_ms == null ? null : this.toBigInt(r.retired_at_ms) })); } listIndexRunsAll(stream) { const rows = this.stmts.listIndexRunsAll.all(stream); return rows.map((r) => ({ run_id: String(r.run_id), stream: String(r.stream), level: Number(r.level), start_segment: Number(r.start_segment), end_segment: Number(r.end_segment), object_key: String(r.object_key), filter_len: Number(r.filter_len), record_count: Number(r.record_count), retired_gen: r.retired_gen == null ? null : Number(r.retired_gen), retired_at_ms: r.retired_at_ms == null ? null : this.toBigInt(r.retired_at_ms) })); } listRetiredIndexRuns(stream) { const rows = this.stmts.listRetiredIndexRuns.all(stream); return rows.map((r) => ({ run_id: String(r.run_id), stream: String(r.stream), level: Number(r.level), start_segment: Number(r.start_segment), end_segment: Number(r.end_segment), object_key: String(r.object_key), filter_len: Number(r.filter_len), record_count: Number(r.record_count), retired_gen: r.retired_gen == null ? null : Number(r.retired_gen), retired_at_ms: r.retired_at_ms == null ? null : this.toBigInt(r.retired_at_ms) })); } insertIndexRun(row) { this.stmts.insertIndexRun.run(row.run_id, row.stream, row.level, row.start_segment, row.end_segment, row.object_key, row.filter_len, row.record_count); } retireIndexRuns(runIds, retiredGen, retiredAtMs) { if (runIds.length === 0) return; const tx = this.db.transaction(() => { for (const runId of runIds) { this.stmts.retireIndexRun.run(retiredGen, retiredAtMs, runId); } }); tx(); } deleteIndexRuns(runIds) { if (runIds.length === 0) return; const tx = this.db.transaction(() => { for (const runId of runIds) { this.stmts.deleteIndexRun.run(runId); } }); tx(); } countUploadedSegments(stream) { const row = this.stmts.countUploadedSegments.get(stream); const maxIdx = row ? Number(row.max_idx) : -1; return maxIdx >= 0 ? maxIdx + 1 : 0; } commitManifest(stream, generation, etag, uploadedAtMs, uploadedThrough) { const tx = this.db.transaction(() => { this.stmts.upsertManifest.run(stream, generation, generation, uploadedAtMs, etag); this.stmts.advanceUploadedThrough.run(uploadedThrough, this.nowMs(), stream); let gcThrough = uploadedThrough; const interp = this.stmts.getStreamInterpreter.get(stream); if (interp) { const interpretedThrough = this.toBigInt(interp.interpreted_through); gcThrough = interpretedThrough < gcThrough ? interpretedThrough : gcThrough; } if (gcThrough < 0n) return; let deleteThrough = gcThrough; if (BASE_WAL_GC_CHUNK_OFFSETS > 0) { const oldest = this.getWalOldestOffset(stream); if (oldest != null) { const maxThrough = oldest + BigInt(BASE_WAL_GC_CHUNK_OFFSETS) - 1n; if (deleteThrough > maxThrough) deleteThrough = maxThrough; } } if (deleteThrough < 0n) return; const bound = this.bindInt(deleteThrough); const stats = this.db.query(`SELECT COALESCE(SUM(payload_len), 0) as bytes, COUNT(*) as rows FROM wal WHERE stream=? AND offset <= ?;`).get(stream, bound); const bytes = this.toBigInt(stats?.bytes ?? 0); const rows = this.toBigInt(stats?.rows ?? 0); if (rows <= 0n) return; this.stmts.deleteWalBeforeOffset.run(stream, bound); const now = this.nowMs(); this.db.query(`UPDATE streams SET wal_bytes = CASE WHEN wal_bytes >= ? THEN wal_bytes - ? ELSE 0 END, wal_rows = CASE WHEN wal_rows >= ? THEN wal_rows - ? ELSE 0 END, updated_at_ms = ? WHERE stream = ?;`).run(bytes, bytes, rows, rows, now, stream); }); tx(); } candidates(minPendingBytes, minPendingRows, maxIntervalMs, limit) { if (maxIntervalMs <= 0n) { return this.stmts.candidateStreamsNoInterval.all(STREAM_FLAG_DELETED | STREAM_FLAG_TOUCH, minPendingBytes, minPendingRows, limit); } const now = this.nowMs(); return this.stmts.candidateStreams.all(STREAM_FLAG_DELETED | STREAM_FLAG_TOUCH, minPendingBytes, minPendingRows, now, maxIntervalMs, limit); } } // src/ingest.ts import { Result as Result2 } from "better-result"; class IngestQueue { cfg; db; stats; gate; memory; metrics; q = []; timer = null; scheduled = false; queuedBytes = 0; lastBacklogWarnMs = 0; stmts; constructor(cfg, db, stats, gate, memory, metrics) { this.cfg = cfg; this.db = db; this.stats = stats; this.gate = gate; this.memory = memory; this.metrics = metrics; this.stmts = { getStream: this.db.db.query(`SELECT stream, epoch, next_offset, last_append_ms, expires_at_ms, stream_flags, content_type, stream_seq, closed, closed_producer_id, closed_producer_epoch, closed_producer_seq FROM streams WHERE stream=? LIMIT 1;`), insertWal: this.db.db.query(`INSERT INTO wal(stream, offset, ts_ms, payload, payload_len, routing_key, content_type, flags) VALUES(?, ?, ?, ?, ?, ?, ?, ?);`), updateStreamAppend: this.db.db.query(`UPDATE streams SET next_offset=?, updated_at_ms=?, last_append_ms=?, pending_rows=pending_rows+?, pending_bytes=pending_bytes+?, wal_rows=wal_rows+?, wal_bytes=wal_bytes+?, stream_seq=?, closed=CASE WHEN ? THEN 1 ELSE closed END, closed_producer_id=CASE WHEN ? THEN ? ELSE closed_producer_id END, closed_producer_epoch=CASE WHEN ? THEN ? ELSE closed_producer_epoch END, closed_producer_seq=CASE WHEN ? THEN ? ELSE closed_producer_seq END WHERE stream=? AND (stream_flags & ?) = 0;`), updateStreamCloseOnly: this.db.db.query(`UPDATE streams SET closed=1, closed_producer_id=?, closed_producer_epoch=?, closed_producer_seq=?, updated_at_ms=?, stream_seq=? WHERE stream=? AND (stream_flags & ?) = 0;`), getProducerState: this.db.db.query(`SELECT epoch, last_seq FROM producer_state WHERE stream=? AND producer_id=? LIMIT 1;`), upsertProducerState: this.db.db.query(`INSERT INTO producer_state(stream, producer_id, epoch, last_seq, updated_at_ms) VALUES(?, ?, ?, ?, ?) ON CONFLICT(stream, producer_id) DO UPDATE SET epoch=excluded.epoch, last_seq=excluded.last_seq, updated_at_ms=excluded.updated_at_ms;`) }; this.timer = setInterval(() => { this.flush(); }, this.cfg.ingestFlushIntervalMs); } stop() { if (this.timer) clearInterval(this.timer); this.timer = null; } append(args, opts) { const bytes = args.rows.reduce((acc, r) => acc + r.payload.byteLength, 0); if (this.memory && !this.memory.shouldAllow()) { this.memory.maybeGc("memory limit"); if (!opts?.bypassBackpressure) { this.memory.maybeHeapSnapshot("memory limit"); if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "memory" }); return Promise.resolve(Result2.err({ kind: "overloaded" })); } } if (!opts?.bypassBackpressure) { if (this.q.length >= this.cfg.ingestMaxQueueRequests || this.queuedBytes + bytes > this.cfg.ingestMaxQueueBytes) { if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "queue" }); return Promise.resolve(Result2.err({ kind: "overloaded" })); } if (this.gate && !this.gate.reserve(bytes)) { if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "backlog" }); this.warnBacklog(); return Promise.resolve(Result2.err({ kind: "overloaded" })); } } this.queuedBytes += bytes; return new Promise((resolve) => { const task = { stream: args.stream, baseAppendMs: args.baseAppendMs, rows: args.rows, contentType: args.contentType ?? null, streamSeq: args.streamSeq ?? null, producer: args.producer ?? null, close: args.close ?? false, reservedBytes: opts?.bypassBackpressure ? 0 : bytes, enqueuedAtMs: this.stats ? Date.now() : undefined, resolve }; if (opts?.priority === "high") this.q.unshift(task); else this.q.push(task); if (!this.scheduled && this.q.length >= this.cfg.ingestMaxBatchRequests) { this.scheduled = true; setTimeout(() => { this.scheduled = false; this.flush(); }, 0); } }); } appendInternal(args) { return this.append(args, { bypassBackpressure: true, priority: "high" }); } getQueueStats() { return { requests: this.q.length, bytes: this.queuedBytes }; } isQueueFull() { return this.q.length >= this.cfg.ingestMaxQueueRequests || this.queuedBytes >= this.cfg.ingestMaxQueueBytes; } warnBacklog() { if (!this.gate) return; const now = Date.now(); if (now - this.lastBacklogWarnMs < 1e4) return; this.lastBacklogWarnMs = now; const current = this.gate.getCurrentBytes(); const max = this.gate.getMaxBytes(); const msg = `[backpressure] local backlog ${formatBytes(current)} exceeds limit ${formatBytes(max)}; rejecting appends (DS_LOCAL_BACKLOG_MAX_BYTES)`; console.warn(msg); } async flush() { if (this.q.length === 0) return; const flushStartMs = Date.now(); let busyWaitMs = 0; const batch = []; let batchBytes = 0; let batchReservedBytes = 0; let drainCount = 0; while (drainCount < this.q.length && batch.length < this.cfg.ingestMaxBatchRequests && batchBytes < this.cfg.ingestMaxBatchBytes) { const t = this.q[drainCount]; batch.push(t); drainCount += 1; for (const r of t.rows) batchBytes += r.payload.byteLength; batchReservedBytes += t.reservedBytes; } if (drainCount > 0) { this.q.splice(0, drainCount); } this.queuedBytes = Math.max(0, this.queuedBytes - batchBytes); let bpOverMs = 0; if (this.stats) { const budgetMs = this.stats.getBackpressureBudgetMs(); const nowMs2 = Date.now(); for (const t of batch) { if (t.enqueuedAtMs == null) continue; const waitMs = Math.max(0, nowMs2 - t.enqueuedAtMs); if (waitMs > budgetMs) { bpOverMs += waitMs - budgetMs; } } } const nowMs = this.db.nowMs(); let perStream = new Map; let perProducer = new Map; let walBytesCommitted = 0; let results = []; const resetAttempt = () => { perStream = new Map; perProducer = new Map; walBytesCommitted = 0; results = new Array(batch.length); }; const tx = this.db.db.transaction(() => { const loadStream = (stream) => { const cached = perStream.get(stream); if (cached) return cached; const row = this.stmts.getStream.get(stream); if (!row || (Number(row.stream_flags) & STREAM_FLAG_DELETED) !== 0) return null; const st = { epoch: Number(row.epoch), nextOffset: BigInt(row.next_offset), lastAppendMs: BigInt(row.last_append_ms), expiresAtMs: row.expires_at_ms == null ? null : BigInt(row.expires_at_ms), streamFlags: Number(row.stream_flags), contentType: String(row.content_type), streamSeq: row.stream_seq == null ? null : String(row.stream_seq), closed: Number(row.closed) !== 0, closedProducerId: row.closed_producer_id == null ? null : String(row.closed_producer_id), closedProducerEpoch: row.closed_producer_epoch == null ? null : Number(row.closed_producer_epoch), closedProducerSeq: row.closed_producer_seq == null ? null : Number(row.closed_producer_seq) }; perStream.set(stream, st); return st; }; const loadProducerState = (stream, producerId) => { const key = `${stream}\x00${producerId}`; if (perProducer.has(key)) return perProducer.get(key); const row = this.stmts.getProducerState.get(stream, producerId); const state = row ? { epoch: Number(row.epoch), lastSeq: Number(row.last_seq) } : null; perProducer.set(key, state); return state; }; const checkProducer = (task) => { const producer = task.producer; const key = `${task.stream}\x00${producer.id}`; const state = loadProducerState(task.stream, producer.id); if (!state) { if (producer.seq !== 0) { return Result2.err({ kind: "producer_epoch_seq" }); } const next = { epoch: producer.epoch, lastSeq: producer.seq }; perProducer.set(key, next); return Result2.ok({ duplicate: false, update: true, epoch: producer.epoch, seq: producer.seq }); } if (producer.epoch < state.epoch) { return Result2.err({ kind: "producer_stale_epoch", producerEpoch: state.epoch }); } if (producer.epoch > state.epoch) { if (producer.seq !== 0) { return Result2.err({ kind: "producer_epoch_seq" }); } const next = { epoch: producer.epoch, lastSeq: producer.seq }; perProducer.set(key, next); return Result2.ok({ duplicate: false, update: true, epoch: producer.epoch, seq: producer.seq }); } if (producer.seq <= state.lastSeq) { return Result2.ok({ duplicate: true, update: false, epoch: state.epoch, seq: state.lastSeq }); } if (producer.seq === state.lastSeq + 1) { const next = { epoch: state.epoch, lastSeq: producer.seq }; perProducer.set(key, next); return Result2.ok({ duplicate: false, update: true, epoch: state.epoch, seq: producer.seq }); } return Result2.err({ kind: "producer_gap", expected: state.lastSeq + 1, received: producer.seq }); }; const checkStreamSeq = (task, st) => { if (task.streamSeq == null) return Result2.ok({ nextSeq: st.streamSeq }); if (st.streamSeq != null && task.streamSeq <= st.streamSeq) { return Result2.err({ kind: "stream_seq", expected: st.streamSeq, received: task.streamSeq }); } return Result2.ok({ nextSeq: task.streamSeq }); }; for (let idx = 0;idx < batch.length; idx++) { const task = batch[idx]; const st = loadStream(task.stream); if (!st) { results[idx] = Result2.err({ kind: "not_found" }); continue; } if (st.expiresAtMs != null && nowMs > st.expiresAtMs) { results[idx] = Result2.err({ kind: "gone" }); continue; } const tailOffset = st.nextOffset - 1n; const isCloseOnly = task.close && task.rows.length === 0; if (st.closed) { if (isCloseOnly) { results[idx] = Result2.ok({ lastOffset: tailOffset, appendedRows: 0, closed: true, duplicate: true }); continue; } if (task.producer && task.close && st.closedProducerId != null && st.closedProducerEpoch != null && st.closedProducerSeq != null && st.closedProducerId === task.producer.id && st.closedProducerEpoch === task.producer.epoch && st.closedProducerSeq === task.producer.seq) { results[idx] = Result2.ok({ lastOffset: tailOffset, appendedRows: 0, closed: true, duplicate: true, producer: { epoch: st.closedProducerEpoch, seq: st.closedProducerSeq } }); continue; } results[idx] = Result2.err({ kind: "closed", lastOffset: tailOffset }); continue; } if (isCloseOnly) { let producerInfo2; let duplicate = false; if (task.producer) { const prodCheck = checkProducer(task); if (Result2.isError(prodCheck)) { results[idx] = Result2.err(prodCheck.error); continue; } duplicate = prodCheck.value.duplicate; producerInfo2 = { epoch: prodCheck.value.epoch, seq: prodCheck.value.seq }; if (prodCheck.value.update) { this.stmts.upsertProducerState.run(task.stream, task.producer.id, prodCheck.value.epoch, prodCheck.value.seq, nowMs); } } if (!duplicate) { const seqCheck2 = checkStreamSeq(task, st); if (Result2.isError(seqCheck2)) { results[idx] = Result2.err(seqCheck2.error); continue; } st.streamSeq = seqCheck2.value.nextSeq; const closedProducer2 = task.producer ?? null; this.stmts.updateStreamCloseOnly.run(closedProducer2 ? closedProducer2.id : null, closedProducer2 ? closedProducer2.epoch : null, closedProducer2 ? closedProducer2.seq : null, nowMs, st.streamSeq, task.stream, STREAM_FLAG_DELETED); st.closed = true; st.closedProducerId = closedProducer2 ? closedProducer2.id : null; st.closedProducerEpoch = closedProducer2 ? closedProducer2.epoch : null; st.closedProducerSeq = closedProducer2 ? closedProducer2.seq : null; } results[idx] = Result2.ok({ lastOffset: tailOffset, appendedRows: 0, closed: st.closed, duplicate, producer: producerInfo2 }); continue; } if (!task.contentType || task.contentType !== st.contentType) { results[idx] = Result2.err({ kind: "content_type_mismatch" }); continue; } let producerInfo; if (task.producer) { const prodCheck = checkProducer(task); if (Result2.isError(prodCheck)) { results[idx] = Result2.err(prodCheck.error); continue; } if (prodCheck.value.duplicate) { results[idx] = Result2.ok({ lastOffset: tailOffset, appendedRows: 0, closed: false, duplicate: true, producer: { epoch: prodCheck.value.epoch, seq: prodCheck.value.seq } }); continue; } producerInfo = { epoch: prodCheck.value.epoch, seq: prodCheck.value.seq }; if (prodCheck.value.update) { this.stmts.upsertProducerState.run(task.stream, task.producer.id, prodCheck.value.epoch, prodCheck.value.seq, nowMs); } } const seqCheck = checkStreamSeq(task, st); if (Result2.isError(seqCheck)) { results[idx] = Result2.err(seqCheck.error); continue; } st.streamSeq = seqCheck.value.nextSeq; let appendMs = task.baseAppendMs; if (appendMs <= st.lastAppendMs) appendMs = st.lastAppendMs + 1n; let offset = st.nextOffset; let totalBytes = 0n; for (let i = 0;i < task.rows.length; i++) { const r = task.rows[i]; const rowAppendMs = appendMs; const payloadLen = r.payload.byteLength; totalBytes += BigInt(payloadLen); this.stmts.insertWal.run(task.stream, offset, rowAppendMs, r.payload, payloadLen, r.routingKey, r.contentType, 0); offset += 1n; } const lastOffset = offset - 1n; st.nextOffset = offset; st.lastAppendMs = appendMs; if (task.close) { st.closed = true; if (task.producer) { st.closedProducerId = task.producer.id; st.closedProducerEpoch = task.producer.epoch; st.closedProducerSeq = task.producer.seq; } else { st.closedProducerId = null; st.closedProducerEpoch = null; st.closedProducerSeq = null; } } const closedProducer = task.close && task.producer ? task.producer : null; const closeFlag = task.close ? 1 : 0; this.stmts.updateStreamAppend.run(st.nextOffset, nowMs, st.lastAppendMs, BigInt(task.rows.length), totalBytes, BigInt(task.rows.length), totalBytes, st.streamSeq, closeFlag, closeFlag, closedProducer ? closedProducer.id : null, closeFlag, closedProducer ? closedProducer.epoch : null, closeFlag, closedProducer ? closedProducer.seq : null, task.stream, STREAM_FLAG_DELETED); walBytesCommitted += Number(totalBytes); results[idx] = Result2.ok({ lastOffset, appendedRows: task.rows.length, closed: task.close, duplicate: false, producer: producerInfo }); } }); const isSqliteBusy = (e) => { const code = String(e?.code ?? ""); const errno = Number(e?.errno ?? -1); return code === "SQLITE_BUSY" || code === "SQLITE_BUSY_SNAPSHOT" || errno === 5 || errno === 517; }; const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); try { const maxBusyMs = Math.max(0, this.cfg.ingestBusyTimeoutMs); const startMs = Date.now(); let attempt = 0; while (true) { resetAttempt(); try { tx(); break; } catch (e) { if (!isSqliteBusy(e)) throw e; if (maxBusyMs <= 0) throw e; const elapsed = Date.now() - startMs; if (elapsed >= maxBusyMs) throw e; const delay = Math.min(200, 5 * 2 ** attempt); attempt += 1; busyWaitMs += delay; await sleep(delay); } } if (this.gate) { const reservedCommitted = Math.min(batchReservedBytes, walBytesCommitted); this.gate.commit(walBytesCommitted, reservedCommitted); const extra = batchReservedBytes - walBytesCommitted; if (extra > 0) this.gate.release(extra); } if (this.stats && walBytesCommitted > 0) this.stats.recordWalCommitBytes(walBytesCommitted); if (this.stats && bpOverMs > 0) this.stats.recordBackpressureOverMs(bpOverMs); for (let i = 0;i < batch.length; i++) batch[i].resolve(results[i] ?? Result2.err({ kind: "internal" })); const elapsedNs = (Date.now() - flushStartMs) * 1e6; if (this.metrics) { this.metrics.record("tieredstore.ingest.flush.latency", elapsedNs, "ns"); if (busyWaitMs > 0) this.metrics.record("tieredstore.ingest.sqlite_busy.wait", busyWaitMs * 1e6, "ns"); } } catch (e) { console.error("ingest tx failed", e); if (this.gate && batchReservedBytes > 0) this.gate.release(batchReservedBytes); for (const t of batch) t.resolve(Result2.err({ kind: "internal" })); const elapsedNs = (Date.now() - flushStartMs) * 1e6; if (this.metrics) { this.metrics.record("tieredstore.ingest.flush.latency", elapsedNs, "ns"); if (busyWaitMs > 0) this.metrics.record("tieredstore.ingest.sqlite_busy.wait", busyWaitMs * 1e6, "ns"); } } } } function formatBytes(bytes) { const units = ["b", "kb", "mb", "gb"]; let value = Math.max(0, bytes); let idx = 0; while (value >= 1024 && idx < units.length - 1) { value /= 1024; idx += 1; } const digits = idx === 0 ? 0 : 1; return `${value.toFixed(digits)}${units[idx]}`; } // src/notifier.ts class StreamNotifier { waiters = new Map; latestSeq = new Map; notify(stream, newEndSeq) { this.latestSeq.set(stream, newEndSeq); const set = this.waiters.get(stream); if (!set || set.size === 0) return; for (const w of Array.from(set)) { if (newEndSeq > w.afterSeq) { set.delete(w); w.resolve(); } } if (set.size === 0) this.waiters.delete(stream); } waitFor(stream, afterSeq, timeoutMs, signal) { if (signal?.aborted) return Promise.resolve(); const latest = this.latestSeq.get(stream); if (latest != null && latest > afterSeq) return Promise.resolve(); return new Promise((resolve) => { let done = false; const set = this.waiters.get(stream) ?? new Set; const cleanup = () => { if (done) return; done = true; const s = this.waiters.get(stream); if (s) { s.delete(waiter); if (s.size === 0) this.waiters.delete(stream); } if (timeoutId) clearTimeout(timeoutId); if (signal) signal.removeEventListener("abort", onAbort); resolve(); }; const waiter = { afterSeq, resolve: cleanup }; set.add(waiter); this.waiters.set(stream, set); const onAbort = () => cleanup(); if (signal) signal.addEventListener("abort", onAbort, { once: true }); let timeoutId = null; if (timeoutMs > 0) { timeoutId = setTimeout(() => { cleanup(); }, timeoutMs); } }); } notifyClose(stream) { const set = this.waiters.get(stream); if (!set || set.size === 0) return; for (const w of Array.from(set)) { set.delete(w); w.resolve(); } if (set.size === 0) this.waiters.delete(stream); } } // src/util/base32_crockford.ts import { Result as Result3 } from "better-result"; var ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; var DECODE_MAP = (() => { const m = {}; for (let i = 0;i < ALPHABET.length; i++) { m[ALPHABET[i]] = i; m[ALPHABET[i].toLowerCase()] = i; } m["O"] = m["o"] = 0; m["I"] = m["i"] = 1; m["L"] = m["l"] = 1; return m; })(); function invalidBase32(message) { return Result3.err({ kind: "invalid_base32", message }); } function encodeCrockfordBase32Fixed26Result(bytes16) { if (bytes16.byteLength !== 16) return invalidBase32(`expected 16 bytes, got ${bytes16.byteLength}`); let n = 0n; for (const b of bytes16) n = n << 8n | BigInt(b); n = n << 2n; let out = ""; for (let i = 0;i < 26; i++) { const shift = 5n * BigInt(25 - i); const idx = Number(n >> shift & 31n); out += ALPHABET[idx]; } return Result3.ok(out); } function decodeCrockfordBase32Fixed26Result(s) { if (s === "-1") return invalidBase32("-1 is a sentinel offset and cannot be decoded as base32"); if (s.length !== 26) return invalidBase32(`expected 26 chars, got ${s.length}`); let n = 0n; for (const ch of s) { const v = DECODE_MAP[ch]; if (v === undefined) return invalidBase32(`invalid base32 char: ${ch}`); n = n << 5n | BigInt(v); } n = n >> 2n; const out = new Uint8Array(16); for (let i = 15;i >= 0; i--) { out[i] = Number(n & 0xffn); n = n >> 8n; } return Result3.ok(out); } // src/util/endian.ts function writeU32BE(dst, offset, value) { const dv = new DataView(dst.buffer, dst.byteOffset, dst.byteLength); dv.setUint32(offset, value >>> 0, false); } function readU32BE(src, offset) { const dv = new DataView(src.buffer, src.byteOffset, src.byteLength); return dv.getUint32(offset, false); } function readU64BE(src, offset) { const dv = new DataView(src.buffer, src.byteOffset, src.byteLength); return dv.getBigUint64(offset, false); } // src/offset.ts import { Result as Result4 } from "better-result"; var DEFAULT_EPOCH = 0; function parseOffsetResult(input) { if (input == null || input === "") { return Result4.err({ kind: "invalid_offset", message: "missing offset" }); } if (input === "-1") return Result4.ok({ kind: "start" }); if (input.length !== 26) { return Result4.err({ kind: "invalid_offset", message: `invalid offset length: ${input.length}` }); } const bytesRes = decodeCrockfordBase32Fixed26Result(input); if (Result4.isError(bytesRes)) return Result4.err({ kind: "invalid_offset", message: bytesRes.error.message }); const bytes = bytesRes.value; const epoch = readU32BE(bytes, 0); const hi = readU32BE(bytes, 4); const lo = readU32BE(bytes, 8); const inBlock = readU32BE(bytes, 12); const rawSeq = BigInt(hi) << 32n | BigInt(lo); const seq = rawSeq - 1n; return Result4.ok({ kind: "seq", epoch, seq, inBlock }); } function parseOffset(input) { const res = parseOffsetResult(input); if (Result4.isError(res)) throw dsError(res.error.message); return res.value; } function encodeOffsetResult(epoch, seq, inBlock = 0) { if (seq < -1n) return Result4.err({ kind: "invalid_offset", message: "invalid offset" }); const bytes = new Uint8Array(16); writeU32BE(bytes, 0, epoch >>> 0); const rawSeq = seq + 1n; const hi = Number(rawSeq >> 32n & 0xffffffffn); const lo = Number(rawSeq & 0xffffffffn); writeU32BE(bytes, 4, hi); writeU32BE(bytes, 8, lo); writeU32BE(bytes, 12, inBlock >>> 0); const encodedRes = encodeCrockfordBase32Fixed26Result(bytes); if (Result4.isError(encodedRes)) return Result4.err({ kind: "invalid_offset", message: encodedRes.error.message }); return Result4.ok(encodedRes.value); } function encodeOffset(epoch, seq, inBlock = 0) { const res = encodeOffsetResult(epoch, seq, inBlock); if (Result4.isError(res)) throw dsError(res.error.message); return res.value; } function canonicalizeOffset(input) { const p = parseOffset(input); if (p.kind === "start") return encodeOffset(DEFAULT_EPOCH, -1n, 0); return encodeOffset(p.epoch, p.seq, p.inBlock); } function offsetToSeqOrNeg1(p) { return p.kind === "start" ? -1n : p.seq; } // src/util/duration.ts import { Result as Result5 } from "better-result"; function parseDurationMsResult(s) { const m = /^([0-9]+)(ms|s|m|h|d)$/.exec(s.trim()); if (!m) return Result5.err({ kind: "invalid_duration", message: `invalid duration: ${s}` }); const n = Number(m[1]); const unit = m[2]; switch (unit) { case "ms": return Result5.ok(n); case "s": return Result5.ok(n * 1000); case "m": return Result5.ok(n * 60 * 1000); case "h": return Result5.ok(n * 60 * 60 * 1000); case "d": return Result5.ok(n * 24 * 60 * 60 * 1000); default: return Result5.err({ kind: "invalid_duration", message: `invalid unit: ${unit}` }); } } // src/metrics.ts class Histogram { maxSamples; samples = []; count = 0; sum = 0; min = Number.POSITIVE_INFINITY; max = Number.NEGATIVE_INFINITY; buckets = {}; constructor(maxSamples = 1024) { this.maxSamples = maxSamples; } add(value) { this.count++; this.sum += value; if (value < this.min) this.min = value; if (value > this.max) this.max = value; const bucket = Math.floor(Math.log2(Math.max(1, value))); const key = String(1 << bucket); this.buckets[key] = (this.buckets[key] ?? 0) + 1; if (this.samples.length < this.maxSamples) { this.samples.push(value); } else { const idx = Math.floor(Math.random() * this.count); if (idx < this.maxSamples) this.samples[idx] = value; } } snapshotAndReset() { const count = this.count; const sum = this.sum; const min = count === 0 ? 0 : this.min; const max = count === 0 ? 0 : this.max; const avg = count === 0 ? 0 : sum / count; const sorted = this.samples.slice().sort((a, b) => a - b); const p = (q) => sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)))]; const p50 = p(0.5); const p95 = p(0.95); const p99 = p(0.99); const buckets = { ...this.buckets }; this.samples = []; this.count = 0; this.sum = 0; this.min = Number.POSITIVE_INFINITY; this.max = Number.NEGATIVE_INFINITY; this.buckets = {}; return { count, sum, min, max, avg, p50, p95, p99, buckets }; } } class MetricSeries { metric; unit; stream; tags; hist = new Histogram; constructor(metric, unit, stream, tags) { this.metric = metric; this.unit = unit; this.stream = stream; this.tags = tags; } } function keyFor(metric, unit, stream, tags) { const tagStr = tags ? JSON.stringify(tags) : ""; return `${metric}|${unit}|${stream ?? ""}|${tagStr}`; } function instanceId() { const host = typeof process !== "undefined" ? process.pid.toString() : "node"; const rand = Math.random().toString(16).slice(2, 8); return `${host}-${rand}`; } class Metrics { startMs = Date.now(); windowStartMs = Date.now(); series = new Map; instance = instanceId(); record(metric, value, unit, tags, stream) { const key = keyFor(metric, unit, stream, tags); let s = this.series.get(key); if (!s) { s = new MetricSeries(metric, unit, stream, tags); this.series.set(key, s); } s.hist.add(value); } recordAppend(bytes, entries) { this.record("tieredstore.append.bytes", bytes, "bytes"); this.record("tieredstore.append.entries", entries, "count"); } recordRead(bytes, entries) { this.record("tieredstore.read.bytes", bytes, "bytes"); this.record("tieredstore.read.entries", entries, "count"); } snapshot() { return { uptime_ms: Date.now() - this.startMs, series: this.series.size }; } flushInterval() { const windowEnd = Date.now(); const intervalMs = windowEnd - this.windowStartMs; const events = []; for (const s of this.series.values()) { const snap = s.hist.snapshotAndReset(); if (snap.count === 0) continue; events.push({ apiVersion: "durable.streams/metrics/v1", kind: "interval", metric: s.metric, unit: s.unit, windowStart: this.windowStartMs, windowEnd, intervalMs, instance: this.instance, stream: s.stream, tags: s.tags, ...snap }); } this.windowStartMs = windowEnd; return events; } } // src/util/time.ts import { Result as Result6 } from "better-result"; function parseTimestampMsResult(input) { const s = input.trim(); if (s === "") return Result6.err({ kind: "empty_timestamp", message: "empty timestamp" }); if (/^[0-9]+$/.test(s)) { return Result6.ok(BigInt(s) / 1000000n); } const d = new Date(s); const ms = d.getTime(); if (Number.isNaN(ms)) return Result6.err({ kind: "invalid_timestamp", message: `invalid timestamp: ${input}` }); return Result6.ok(BigInt(ms)); } // src/util/cleanup.ts import { existsSync, readdirSync, unlinkSync } from "node:fs"; import { join } from "node:path"; function cleanupTempSegments(rootDir) { const base = join(rootDir, "local"); if (!existsSync(base)) return; const walk = (dir) => { for (const entry of readdirSync(dir, { withFileTypes: true })) { const full = join(dir, entry.name); if (entry.isDirectory()) { walk(full); } else if (entry.isFile() && entry.name.endsWith(".tmp")) { try { unlinkSync(full); } catch {} } } }; walk(base); } // src/metrics_emitter.ts class MetricsEmitter { metrics; ingest; intervalMs; timer = null; constructor(metrics, ingest, intervalMs) { this.metrics = metrics; this.ingest = ingest; this.intervalMs = intervalMs; } start() { if (this.intervalMs <= 0 || this.timer) return; this.timer = setInterval(() => { this.flush(); }, this.intervalMs); } stop() { if (this.timer) clearInterval(this.timer); this.timer = null; } async flush() { const queue = this.ingest.getQueueStats(); this.metrics.record("tieredstore.ingest.queue.bytes", queue.bytes, "bytes"); this.metrics.record("tieredstore.ingest.queue.requests", queue.requests, "count"); const events = this.metrics.flushInterval(); if (events.length === 0) return; const rows = events.map((e) => ({ routingKey: e.stream ? new TextEncoder().encode(e.stream) : null, contentType: "application/json", payload: new TextEncoder().encode(JSON.stringify(e)) })); try { await this.ingest.appendInternal({ stream: "__stream_metrics__", baseAppendMs: BigInt(Date.now()), rows, contentType: "application/json" }); } catch {} } } // src/schema/registry.ts import Ajv from "ajv"; import { createHash } from "node:crypto"; import { Result as Result11 } from "better-result"; // src/util/lru.ts class LruCache { maxEntries; map = new Map; constructor(maxEntries) { if (maxEntries <= 0) throw dsError("maxEntries must be > 0"); this.maxEntries = maxEntries; } get size() { return this.map.size; } get(key) { const value = this.map.get(key); if (value === undefined) return; this.map.delete(key); this.map.set(key, value); return value; } set(key, value) { if (this.map.has(key)) this.map.delete(key); this.map.set(key, value); while (this.map.size > this.maxEntries) { const oldest = this.map.keys().next(); if (oldest.done) break; this.map.delete(oldest.value); } } has(key) { return this.map.has(key); } delete(key) { return this.map.delete(key); } clear() { this.map.clear(); } } // src/schema/lens_schema.ts var DURABLE_LENS_V1_SCHEMA = { $schema: "https://json-schema.org/draft/2020-12/schema", $id: "https://example.com/schemas/durable-lens-v1.schema.json", title: "Durable Stream Lens Spec", type: "object", additionalProperties: false, required: ["apiVersion", "schema", "from", "to", "ops"], properties: { apiVersion: { type: "string", const: "durable.lens/v1" }, schema: { type: "string", minLength: 1, description: "Logical stream schema/type name (e.g., 'Task')." }, from: { type: "integer", minimum: 0, description: "Source schema version." }, to: { type: "integer", minimum: 0, description: "Target schema version." }, description: { type: "string" }, ops: { type: "array", minItems: 1, items: { $ref: "#/$defs/op" } } }, $defs: { jsonPointer: { type: "string", description: "RFC 6901 JSON Pointer. Empty string refers to the document root.", pattern: "^(?:/(?:[^~/]|~0|~1)*)*$" }, jsonScalar: { description: "JSON scalar value.", type: ["string", "number", "integer", "boolean", "null"] }, embeddedJsonSchema: { description: "An embedded JSON Schema fragment (object or boolean).", anyOf: [{ type: "object" }, { type: "boolean" }] }, mapTransform: { type: "object", additionalProperties: false, required: ["map"], properties: { map: { type: "object", description: "Mapping table for values. Keys must be strings; values are JSON scalars.", additionalProperties: { $ref: "#/$defs/jsonScalar" } }, default: { $ref: "#/$defs/jsonScalar", description: "Default output value used when input is not found in map." } } }, builtinTransform: { type: "object", additionalProperties: false, required: ["builtin"], properties: { builtin: { type: "string", minLength: 1, description: "Name of a built-in, version-stable transform implemented by the system." } } }, convertTransform: { description: "Restricted conversion: either a total mapping table (+ optional default) or a built-in transform.", oneOf: [{ $ref: "#/$defs/mapTransform" }, { $ref: "#/$defs/builtinTransform" }] }, opRename: { type: "object", additionalProperties: false, required: ["op", "from", "to"], properties: { op: { const: "rename" }, from: { $ref: "#/$defs/jsonPointer" }, to: { $ref: "#/$defs/jsonPointer" } } }, opCopy: { type: "object", additionalProperties: false, required: ["op", "from", "to"], properties: { op: { const: "copy" }, from: { $ref: "#/$defs/jsonPointer" }, to: { $ref: "#/$defs/jsonPointer" } } }, opAdd: { type: "object", additionalProperties: false, required: ["op", "path", "schema"], properties: { op: { const: "add" }, path: { $ref: "#/$defs/jsonPointer" }, schema: { $ref: "#/$defs/embeddedJsonSchema" }, default: { description: "Default value to insert if the field is missing. If omitted, the runtime may derive a default when possible.", type: ["object", "array", "string", "number", "integer", "boolean", "null"] } } }, opRemove: { type: "object", additionalProperties: false, required: ["op", "path", "schema"], properties: { op: { const: "remove" }, path: { $ref: "#/$defs/jsonPointer" }, schema: { $ref: "#/$defs/embeddedJsonSchema", description: "Schema of the removed field (required so the transformation remains declarative and reversible/validatable)." }, default: { description: "Default to use when reconstructing the field (e.g., during backward reasoning/validation).", type: ["object", "array", "string", "number", "integer", "boolean", "null"] } } }, opHoist: { type: "object", additionalProperties: false, required: ["op", "host", "name", "to"], properties: { op: { const: "hoist" }, host: { $ref: "#/$defs/jsonPointer", description: "Pointer to an object field that contains the nested value." }, name: { type: "string", minLength: 1, description: "Field name inside host to move outward." }, to: { $ref: "#/$defs/jsonPointer", description: "Destination pointer for the hoisted value." }, removeFromHost: { type: "boolean", default: true, description: "If true, remove the nested field from the host after hoisting." } } }, opPlunge: { type: "object", additionalProperties: false, required: ["op", "from", "host", "name"], properties: { op: { const: "plunge" }, from: { $ref: "#/$defs/jsonPointer", description: "Pointer to the source field to move inward." }, host: { $ref: "#/$defs/jsonPointer", description: "Pointer to the destination object field." }, name: { type: "string", minLength: 1, description: "Field name inside host to receive the value." }, createHost: { type: "boolean", default: true, description: "If true, create the destination host object if missing." }, removeFromSource: { type: "boolean", default: true, description: "If true, remove the source field after plunging." } } }, opWrap: { type: "object", additionalProperties: false, required: ["op", "path", "mode"], properties: { op: { const: "wrap" }, path: { $ref: "#/$defs/jsonPointer" }, mode: { type: "string", enum: ["singleton"], description: "singleton: x -> [x]" }, reverseMode: { type: "string", enum: ["first"], default: "first", description: "When reversing array->scalar, choose 'first'." } } }, opHead: { type: "object", additionalProperties: false, required: ["op", "path"], properties: { op: { const: "head" }, path: { $ref: "#/$defs/jsonPointer" }, reverseMode: { type: "string", enum: ["singleton"], default: "singleton", description: "When reversing scalar->array, wrap as [scalar]." } } }, opConvert: { type: "object", additionalProperties: false, required: ["op", "path", "fromType", "toType", "forward", "backward"], properties: { op: { const: "convert" }, path: { $ref: "#/$defs/jsonPointer" }, fromType: { type: "string", enum: ["string", "number", "integer", "boolean", "null", "object", "array"] }, toType: { type: "string", enum: ["string", "number", "integer", "boolean", "null", "object", "array"] }, forward: { $ref: "#/$defs/convertTransform" }, backward: { $ref: "#/$defs/convertTransform" } } }, opIn: { type: "object", additionalProperties: false, required: ["op", "path", "ops"], properties: { op: { const: "in" }, path: { $ref: "#/$defs/jsonPointer" }, ops: { type: "array", minItems: 1, items: { $ref: "#/$defs/op" } } } }, opMap: { type: "object", additionalProperties: false, required: ["op", "path", "ops"], properties: { op: { const: "map" }, path: { $ref: "#/$defs/jsonPointer" }, ops: { type: "array", minItems: 1, items: { $ref: "#/$defs/op" } } } }, op: { description: "A single lens operation.", oneOf: [ { $ref: "#/$defs/opRename" }, { $ref: "#/$defs/opCopy" }, { $ref: "#/$defs/opAdd" }, { $ref: "#/$defs/opRemove" }, { $ref: "#/$defs/opHoist" }, { $ref: "#/$defs/opPlunge" }, { $ref: "#/$defs/opWrap" }, { $ref: "#/$defs/opHead" }, { $ref: "#/$defs/opConvert" }, { $ref: "#/$defs/opIn" }, { $ref: "#/$defs/opMap" } ] } } }; // src/lens/lens.ts import { Result as Result8 } from "better-result"; // src/util/json_pointer.ts import { Result as Result7 } from "better-result"; function parseJsonPointerResult(ptr) { if (ptr === "") return Result7.ok([]); if (!ptr.startsWith("/")) return Result7.err({ kind: "invalid_json_pointer", message: "invalid json pointer" }); return Result7.ok(ptr.split("/").slice(1).map((seg) => seg.replace(/~1/g, "/").replace(/~0/g, "~"))); } function isArrayIndex(seg) { return seg !== "" && /^[0-9]+$/.test(seg); } function getChild(container, seg) { if (Array.isArray(container) && isArrayIndex(seg)) { return container[Number(seg)]; } if (container && typeof container === "object") { return container[seg]; } return; } function resolvePointerResult(doc, ptr) { const segmentsRes = parseJsonPointerResult(ptr); if (Result7.isError(segmentsRes)) return Result7.err(segmentsRes.error); const segments = segmentsRes.value; if (segments.length === 0) { return Result7.ok({ parent: null, key: null, value: doc, exists: true }); } let cur = doc; for (let i = 0;i < segments.length - 1; i++) { cur = getChild(cur, segments[i]); if (cur === undefined) return Result7.ok({ parent: null, key: null, value: undefined, exists: false }); } const key = segments[segments.length - 1]; const value = cur === undefined ? undefined : getChild(cur, key); return Result7.ok({ parent: cur, key, value, exists: value !== undefined }); } // src/lens/lens.ts function invalidLensApply(message) { return Result8.err({ kind: "invalid_lens_ops", message }); } function resolveSegments(doc, segments) { if (segments.length === 0) return { parent: null, key: null, value: doc, exists: true }; let cur = doc; for (let i = 0;i < segments.length - 1; i++) { cur = getChild2(cur, segments[i]); if (cur === undefined) return { parent: null, key: null, value: undefined, exists: false }; } const key = segments[segments.length - 1]; const value = cur === undefined ? undefined : getChild2(cur, key); return { parent: cur, key, value, exists: value !== undefined }; } function getChild2(container, seg) { if (Array.isArray(container)) { const idx = Number(seg); if (!Number.isInteger(idx)) return; return container[idx]; } if (container && typeof container === "object") return container[seg]; return; } function setChildResult(container, seg, value) { if (Array.isArray(container)) { const idx = Number(seg); if (!Number.isInteger(idx) || idx < 0 || idx >= container.length) return invalidLensApply("array index out of bounds"); container[idx] = value; return Result8.ok(undefined); } if (container && typeof container === "object") { container[seg] = value; return Result8.ok(undefined); } return invalidLensApply("invalid parent"); } function deleteChildResult(container, seg) { if (Array.isArray(container)) { const idx = Number(seg); if (!Number.isInteger(idx) || idx < 0 || idx >= container.length) return invalidLensApply("array index out of bounds"); container.splice(idx, 1); return Result8.ok(undefined); } if (container && typeof container === "object") { delete container[seg]; return Result8.ok(undefined); } return invalidLensApply("invalid parent"); } function setAtResult(doc, segments, value, opts) { if (segments.length === 0) return Result8.ok(value); let cur = doc; for (let i = 0;i < segments.length - 1; i++) { const seg = segments[i]; let next = getChild2(cur, seg); if (next === undefined) { if (!opts?.createParents) return invalidLensApply("missing parent"); next = {}; const setRes = setChildResult(cur, seg, next); if (Result8.isError(setRes)) return setRes; } cur = next; } const setLeafRes = setChildResult(cur, segments[segments.length - 1], value); if (Result8.isError(setLeafRes)) return setLeafRes; return Result8.ok(doc); } function deleteAtResult(doc, segments) { if (segments.length === 0) return invalidLensApply("cannot delete document root"); let cur = doc; for (let i = 0;i < segments.length - 1; i++) { cur = getChild2(cur, segments[i]); if (cur === undefined) return invalidLensApply("missing parent"); } const deleteRes = deleteChildResult(cur, segments[segments.length - 1]); if (Result8.isError(deleteRes)) return deleteRes; return Result8.ok(doc); } function coerceTypeNameResult(value) { if (value === null) return Result8.ok("null"); if (Array.isArray(value)) return Result8.ok("array"); switch (typeof value) { case "string": return Result8.ok("string"); case "number": return Result8.ok(Number.isInteger(value) ? "integer" : "number"); case "boolean": return Result8.ok("boolean"); case "object": return Result8.ok("object"); default: return invalidLensApply("invalid json type"); } } function ensureTypeResult(value, expected) { const actualRes = coerceTypeNameResult(value); if (Result8.isError(actualRes)) return actualRes; const actual = actualRes.value; if (expected === "number" && (actual === "number" || actual === "integer")) return Result8.ok(undefined); if (actual !== expected) return invalidLensApply(`type mismatch: expected ${expected}, got ${actual}`); return Result8.ok(undefined); } function applyTransformResult(transform, value) { if (transform.builtin) { const name = transform.builtin; return applyBuiltinResult(name, value); } const t = transform; const key = String(value); if (Object.prototype.hasOwnProperty.call(t.map, key)) return Result8.ok(t.map[key]); if (Object.prototype.hasOwnProperty.call(t, "default")) return Result8.ok(t.default); return invalidLensApply("convert map missing key and default"); } function applyBuiltinResult(name, value) { switch (name) { case "lowercase": if (typeof value !== "string") return invalidLensApply("builtin lowercase expects string"); return Result8.ok(value.toLowerCase()); case "uppercase": if (typeof value !== "string") return invalidLensApply("builtin uppercase expects string"); return Result8.ok(value.toUpperCase()); case "string_to_int": { if (typeof value !== "string") return invalidLensApply("builtin string_to_int expects string"); const n = Number.parseInt(value, 10); if (!Number.isFinite(n)) return invalidLensApply("builtin string_to_int invalid"); return Result8.ok(n); } case "int_to_string": if (typeof value !== "number" || !Number.isInteger(value)) return invalidLensApply("builtin int_to_string expects integer"); return Result8.ok(String(value)); case "rfc3339_to_unix_millis": { if (typeof value !== "string") return invalidLensApply("builtin rfc3339_to_unix_millis expects string"); const ms = new Date(value).getTime(); if (Number.isNaN(ms)) return invalidLensApply("builtin rfc3339_to_unix_millis invalid"); return Result8.ok(ms); } case "unix_millis_to_rfc3339": if (typeof value !== "number" || !Number.isFinite(value)) return invalidLensApply("builtin unix_millis_to_rfc3339 expects number"); return Result8.ok(new Date(value).toISOString()); default: return invalidLensApply(`unknown builtin: ${name}`); } } function compileLensResult(lens) { const opsRes = compileOpsResult(lens.ops); if (Result8.isError(opsRes)) return opsRes; return Result8.ok({ schema: lens.schema, from: lens.from, to: lens.to, ops: opsRes.value }); } function invalidLens(message) { return Result8.err({ kind: "invalid_lens", message }); } function parsePointer(pointer, field) { const res = parseJsonPointerResult(pointer); if (Result8.isError(res)) return invalidLens(`${field} ${res.error.message}`); return Result8.ok(res.value); } function compileOpsResult(ops) { const compiled = []; for (const op of ops) { switch (op.op) { case "rename": { const fromRes = parsePointer(op.from, "rename.from:"); if (Result8.isError(fromRes)) return fromRes; const toRes = parsePointer(op.to, "rename.to:"); if (Result8.isError(toRes)) return toRes; compiled.push({ op: "rename", from: fromRes.value, to: toRes.value }); break; } case "copy": { const fromRes = parsePointer(op.from, "copy.from:"); if (Result8.isError(fromRes)) return fromRes; const toRes = parsePointer(op.to, "copy.to:"); if (Result8.isError(toRes)) return toRes; compiled.push({ op: "copy", from: fromRes.value, to: toRes.value }); break; } case "add": { const pathRes = parsePointer(op.path, "add.path:"); if (Result8.isError(pathRes)) return pathRes; compiled.push({ op: "add", path: pathRes.value, default: op.default }); break; } case "remove": { const pathRes = parsePointer(op.path, "remove.path:"); if (Result8.isError(pathRes)) return pathRes; compiled.push({ op: "remove", path: pathRes.value }); break; } case "hoist": { const hostRes = parsePointer(op.host, "hoist.host:"); if (Result8.isError(hostRes)) return hostRes; const toRes = parsePointer(op.to, "hoist.to:"); if (Result8.isError(toRes)) return toRes; compiled.push({ op: "hoist", host: hostRes.value, name: op.name, to: toRes.value, removeFromHost: op.removeFromHost !== false }); break; } case "plunge": { const fromRes = parsePointer(op.from, "plunge.from:"); if (Result8.isError(fromRes)) return fromRes; const hostRes = parsePointer(op.host, "plunge.host:"); if (Result8.isError(hostRes)) return hostRes; compiled.push({ op: "plunge", from: fromRes.value, host: hostRes.value, name: op.name, createHost: op.createHost !== false, removeFromSource: op.removeFromSource !== false }); break; } case "wrap": { const pathRes = parsePointer(op.path, "wrap.path:"); if (Result8.isError(pathRes)) return pathRes; compiled.push({ op: "wrap", path: pathRes.value }); break; } case "head": { const pathRes = parsePointer(op.path, "head.path:"); if (Result8.isError(pathRes)) return pathRes; compiled.push({ op: "head", path: pathRes.value }); break; } case "convert": { const pathRes = parsePointer(op.path, "convert.path:"); if (Result8.isError(pathRes)) return pathRes; compiled.push({ op: "convert", path: pathRes.value, fromType: op.fromType, toType: op.toType, forward: op.forward }); break; } case "in": { const pathRes = parsePointer(op.path, "in.path:"); if (Result8.isError(pathRes)) return pathRes; const nestedRes = compileOpsResult(op.ops); if (Result8.isError(nestedRes)) return nestedRes; compiled.push({ op: "in", path: pathRes.value, ops: nestedRes.value }); break; } case "map": { const pathRes = parsePointer(op.path, "map.path:"); if (Result8.isError(pathRes)) return pathRes; const nestedRes = compileOpsResult(op.ops); if (Result8.isError(nestedRes)) return nestedRes; compiled.push({ op: "map", path: pathRes.value, ops: nestedRes.value }); break; } default: { return invalidLens(`unknown op: ${op.op}`); } } } return Result8.ok(compiled); } function applyCompiledLensResult(lens, doc) { return applyOpsResult(doc, lens.ops); } function applyLensChainResult(lenses, doc) { let cur = doc; for (const l of lenses) { const stepRes = applyCompiledLensResult(l, cur); if (Result8.isError(stepRes)) return stepRes; cur = stepRes.value; } return Result8.ok(cur); } function applyOpsResult(doc, ops) { let root = doc; for (const op of ops) { switch (op.op) { case "rename": { const src = resolveSegments(root, op.from); if (!src.exists) return invalidLensApply("rename missing source"); const setRes = setAtResult(root, op.to, src.value); if (Result8.isError(setRes)) return setRes; const deleteRes = deleteAtResult(setRes.value, op.from); if (Result8.isError(deleteRes)) return deleteRes; root = deleteRes.value; break; } case "copy": { const src = resolveSegments(root, op.from); if (!src.exists) return invalidLensApply("copy missing source"); const setRes = setAtResult(root, op.to, src.value); if (Result8.isError(setRes)) return setRes; root = setRes.value; break; } case "add": { const setRes = setAtResult(root, op.path, op.default); if (Result8.isError(setRes)) return setRes; root = setRes.value; break; } case "remove": { const dst = resolveSegments(root, op.path); if (!dst.exists) return invalidLensApply("remove missing path"); const deleteRes = deleteAtResult(root, op.path); if (Result8.isError(deleteRes)) return deleteRes; root = deleteRes.value; break; } case "hoist": { const host = resolveSegments(root, op.host); if (!host.exists || !host.value || typeof host.value !== "object" || Array.isArray(host.value)) { return invalidLensApply("hoist host missing or not object"); } if (!(op.name in host.value)) return invalidLensApply("hoist missing name"); const value = host.value[op.name]; const setRes = setAtResult(root, op.to, value); if (Result8.isError(setRes)) return setRes; root = setRes.value; if (op.removeFromHost) delete host.value[op.name]; break; } case "plunge": { const src = resolveSegments(root, op.from); if (!src.exists) return invalidLensApply("plunge missing source"); let host = resolveSegments(root, op.host); if (!host.exists) { if (!op.createHost) return invalidLensApply("plunge host missing"); const setHostRes = setAtResult(root, op.host, {}, { createParents: true }); if (Result8.isError(setHostRes)) return setHostRes; root = setHostRes.value; host = resolveSegments(root, op.host); } if (!host.value || typeof host.value !== "object" || Array.isArray(host.value)) { return invalidLensApply("plunge host not object"); } host.value[op.name] = src.value; if (op.removeFromSource) { const deleteRes = deleteAtResult(root, op.from); if (Result8.isError(deleteRes)) return deleteRes; root = deleteRes.value; } break; } case "wrap": { const dst = resolveSegments(root, op.path); if (!dst.exists) return invalidLensApply("wrap missing path"); const setRes = setAtResult(root, op.path, [dst.value]); if (Result8.isError(setRes)) return setRes; root = setRes.value; break; } case "head": { const dst = resolveSegments(root, op.path); if (!dst.exists) return invalidLensApply("head missing path"); if (!Array.isArray(dst.value) || dst.value.length === 0) return invalidLensApply("head expects non-empty array"); const setRes = setAtResult(root, op.path, dst.value[0]); if (Result8.isError(setRes)) return setRes; root = setRes.value; break; } case "convert": { const dst = resolveSegments(root, op.path); if (!dst.exists) return invalidLensApply("convert missing path"); const ensureFromRes = ensureTypeResult(dst.value, op.fromType); if (Result8.isError(ensureFromRes)) return ensureFromRes; const outRes = applyTransformResult(op.forward, dst.value); if (Result8.isError(outRes)) return outRes; const ensureToRes = ensureTypeResult(outRes.value, op.toType); if (Result8.isError(ensureToRes)) return ensureToRes; const setRes = setAtResult(root, op.path, outRes.value); if (Result8.isError(setRes)) return setRes; root = setRes.value; break; } case "in": { const dst = resolveSegments(root, op.path); if (!dst.exists) return invalidLensApply("in missing path"); if (!dst.value || typeof dst.value !== "object" || Array.isArray(dst.value)) return invalidLensApply("in expects object"); const nestedRes = applyOpsResult(dst.value, op.ops); if (Result8.isError(nestedRes)) return nestedRes; break; } case "map": { const dst = resolveSegments(root, op.path); if (!dst.exists) return invalidLensApply("map missing path"); if (!Array.isArray(dst.value)) return invalidLensApply("map expects array"); for (const item of dst.value) { const nestedRes = applyOpsResult(item, op.ops); if (Result8.isError(nestedRes)) return nestedRes; } break; } default: return invalidLensApply(`unknown op: ${op.op}`); } } return Result8.ok(root); } function lensFromJson(raw) { return raw; } // src/schema/proof.ts import { Result as Result9 } from "better-result"; function invalidLensProof(message) { return Result9.err({ kind: "invalid_lens_proof", message }); } function invalidLensDefaults(message) { return Result9.err({ kind: "invalid_lens_defaults", message }); } function parsePointerResult(ptr, kind) { const res = parseJsonPointerResult(ptr); if (Result9.isError(res)) { return Result9.err({ kind, message: res.error.message }); } return Result9.ok(res.value); } function isObjectSchema(schema) { if (schema === true) return true; if (schema === false || schema == null) return false; const t = schema.type; if (t === undefined) return true; if (Array.isArray(t)) return t.includes("object"); return t === "object"; } function isArraySchema(schema) { if (schema === true) return true; if (schema === false || schema == null) return false; const t = schema.type; if (t === undefined) return true; if (Array.isArray(t)) return t.includes("array"); return t === "array"; } function getTypeSet(schema) { if (schema === true) return null; if (schema === false || schema == null) return new Set; if (schema.const !== undefined) return new Set([inferType(schema.const)]); if (schema.enum) return new Set(schema.enum.map((v) => inferType(v))); const t = schema.type; if (t === undefined) return null; if (Array.isArray(t)) return new Set(t); return new Set([t]); } function inferType(value) { if (value === null) return "null"; if (Array.isArray(value)) return "array"; switch (typeof value) { case "string": return "string"; case "number": return Number.isInteger(value) ? "integer" : "number"; case "boolean": return "boolean"; case "object": return "object"; default: return "unknown"; } } function typeSubset(src, dest) { if (src === dest) return true; if (src === "integer" && dest === "number") return true; return false; } function isSchemaCompatible(src, dest) { if (dest === true) return true; if (dest === false || dest == null) return false; if (src && src.const !== undefined) { return valueConformsToSchema(src.const, dest); } if (src && Array.isArray(src.enum)) { return src.enum.every((v) => valueConformsToSchema(v, dest)); } const srcTypes = getTypeSet(src); const destTypes = getTypeSet(dest); if (srcTypes && destTypes) { for (const t of srcTypes) { let ok = false; for (const d of destTypes) { if (typeSubset(t, d)) { ok = true; break; } } if (!ok) return false; } return true; } if (dest.const !== undefined || Array.isArray(dest.enum)) return false; return true; } function valueConformsToSchema(value, schema) { if (schema === true) return true; if (schema === false || schema == null) return false; if (schema.const !== undefined) return deepEqual(value, schema.const); if (Array.isArray(schema.enum)) return schema.enum.some((v) => deepEqual(v, value)); const t = getTypeSet(schema); if (t) { const vt = inferType(value); for (const allowed of t) { if (typeSubset(vt, allowed)) return true; } return false; } return true; } function deepEqual(a, b) { return JSON.stringify(a) === JSON.stringify(b); } function getPropertySchema(schema, prop, requireExplicit) { if (!schema || schema === false) return { schema, required: false, exists: false, explicit: false }; if (!isObjectSchema(schema)) return { schema, required: false, exists: false, explicit: false }; const props = schema.properties ?? {}; const required = Array.isArray(schema.required) && schema.required.includes(prop); if (Object.prototype.hasOwnProperty.call(props, prop)) { return { schema: props[prop], required, exists: true, explicit: true }; } if (requireExplicit) return { schema, required, exists: false, explicit: false }; if (schema.additionalProperties === false) return { schema, required, exists: false, explicit: false }; if (schema.additionalProperties && schema.additionalProperties !== true) { return { schema: schema.additionalProperties, required, exists: true, explicit: false }; } return { schema: true, required, exists: true, explicit: false }; } function getItemsSchema(schema) { if (!schema || schema === false) return schema; if (!isArraySchema(schema)) return schema; if (schema.items !== undefined) return schema.items; return true; } function getPathInfo(schema, segments, requireExplicit) { if (segments.length === 0) return { schema, required: true, exists: true, explicit: true }; let cur = schema; let required = true; for (let i = 0;i < segments.length; i++) { const seg = segments[i]; if (/^[0-9]+$/.test(seg)) { if (!isArraySchema(cur)) return { schema: cur, required: false, exists: false, explicit: false }; cur = getItemsSchema(cur); continue; } const info = getPropertySchema(cur, seg, requireExplicit); if (!info.exists) return info; required = required && info.required; cur = info.schema; } return { schema: cur, required, exists: true, explicit: true }; } function ensureParentRequiredResult(schema, segments, requireExplicit, kind) { if (segments.length === 0) return Result9.ok(undefined); let cur = schema; for (let i = 0;i < segments.length - 1; i++) { const seg = segments[i]; const info = getPropertySchema(cur, seg, requireExplicit); if (!info.exists || !info.required) { return Result9.err({ kind, message: "parent path not required" }); } if (!isObjectSchema(info.schema)) { return Result9.err({ kind, message: "parent path not object" }); } cur = info.schema; } return Result9.ok(undefined); } function isPathRequired(schema, segments) { const info = getPathInfo(schema, segments, false); return info.exists && info.required; } function deriveDefault(schema) { if (schema === false || schema == null) return; if (schema.default !== undefined) return schema.default; const types = getTypeSet(schema); if (types) { if (types.has("object")) return {}; if (types.has("array")) return []; } return; } function ensureEnumOrConstResult(kind, schema) { if (schema && (schema.const !== undefined || Array.isArray(schema.enum))) return Result9.ok(undefined); return Result9.err({ kind, message: "schema must be enum/const for non-total transform" }); } function samePointer(a, b) { if (a.length !== b.length) return false; for (let i = 0;i < a.length; i++) if (a[i] !== b[i]) return false; return true; } function hasWrapAfterResult(ops, pathSeg) { for (const op of ops) { if (op.op !== "wrap") continue; const wrapPathRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(wrapPathRes)) return wrapPathRes; if (samePointer(wrapPathRes.value, pathSeg)) return Result9.ok(true); } return Result9.ok(false); } function findRenameToResult(priorOps, pathSeg) { for (let i = priorOps.length - 1;i >= 0; i--) { const op = priorOps[i]; if (op.op !== "rename") continue; const toRes = parsePointerResult(op.to, "invalid_lens_proof"); if (Result9.isError(toRes)) return toRes; if (samePointer(toRes.value, pathSeg)) return Result9.ok(op); } return Result9.ok(null); } function validateOpResult(oldSchema, newSchema, op, priorOps, remainingOps) { switch (op.op) { case "rename": { const fromSegRes = parsePointerResult(op.from, "invalid_lens_proof"); if (Result9.isError(fromSegRes)) return fromSegRes; const toSegRes = parsePointerResult(op.to, "invalid_lens_proof"); if (Result9.isError(toSegRes)) return toSegRes; const fromSeg = fromSegRes.value; const toSeg = toSegRes.value; const src = getPathInfo(oldSchema, fromSeg, false); if (!src.exists || !src.required) return invalidLensProof("rename source not required"); const parentRes = ensureParentRequiredResult(oldSchema, toSeg, false, "invalid_lens_proof"); if (Result9.isError(parentRes)) return parentRes; const dest = getPathInfo(newSchema, toSeg, false); if (!dest.exists) return invalidLensProof("rename dest missing in new schema"); if (!isSchemaCompatible(src.schema, dest.schema)) { if (isArraySchema(dest.schema)) { const hasWrapRes = hasWrapAfterResult(remainingOps, toSeg); if (Result9.isError(hasWrapRes)) return hasWrapRes; if (hasWrapRes.value) { const items = getItemsSchema(dest.schema); if (!isSchemaCompatible(src.schema, items)) { return invalidLensProof("rename schema incompatible"); } return Result9.ok(undefined); } } return invalidLensProof("rename schema incompatible"); } return Result9.ok(undefined); } case "copy": { const fromSegRes = parsePointerResult(op.from, "invalid_lens_proof"); if (Result9.isError(fromSegRes)) return fromSegRes; const toSegRes = parsePointerResult(op.to, "invalid_lens_proof"); if (Result9.isError(toSegRes)) return toSegRes; const fromSeg = fromSegRes.value; const toSeg = toSegRes.value; const src = getPathInfo(oldSchema, fromSeg, false); if (!src.exists || !src.required) return invalidLensProof("copy source not required"); const parentRes = ensureParentRequiredResult(oldSchema, toSeg, false, "invalid_lens_proof"); if (Result9.isError(parentRes)) return parentRes; const dest = getPathInfo(newSchema, toSeg, false); if (!dest.exists) return invalidLensProof("copy dest missing in new schema"); if (!isSchemaCompatible(src.schema, dest.schema)) return invalidLensProof("copy schema incompatible"); return Result9.ok(undefined); } case "add": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; const parentRes = ensureParentRequiredResult(oldSchema, pathSeg, true, "invalid_lens_proof"); if (Result9.isError(parentRes)) return parentRes; const dest = getPathInfo(newSchema, pathSeg, true); if (!dest.exists) return invalidLensProof("add dest missing in new schema"); const def = op.default ?? deriveDefault(dest.schema); if (def === undefined) return invalidLensProof("add missing default"); if (!valueConformsToSchema(def, dest.schema)) return invalidLensProof("add default invalid"); return Result9.ok(undefined); } case "remove": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; const src = getPathInfo(oldSchema, pathSeg, true); if (!src.exists || !src.required) return invalidLensProof("remove path not required"); if (isPathRequired(newSchema, pathSeg)) return invalidLensProof("remove path still required in new schema"); return Result9.ok(undefined); } case "hoist": { const hostSegRes = parsePointerResult(op.host, "invalid_lens_proof"); if (Result9.isError(hostSegRes)) return hostSegRes; const toSegRes = parsePointerResult(op.to, "invalid_lens_proof"); if (Result9.isError(toSegRes)) return toSegRes; const hostSeg = hostSegRes.value; const toSeg = toSegRes.value; const host = getPathInfo(oldSchema, hostSeg, true); if (!host.exists || !host.required || !isObjectSchema(host.schema)) return invalidLensProof("hoist host invalid"); const child = getPropertySchema(host.schema, op.name, true); if (!child.exists || !child.required) return invalidLensProof("hoist name missing/optional"); const parentRes = ensureParentRequiredResult(oldSchema, toSeg, true, "invalid_lens_proof"); if (Result9.isError(parentRes)) return parentRes; const dest = getPathInfo(newSchema, toSeg, true); if (!dest.exists) return invalidLensProof("hoist dest missing in new schema"); if (!isSchemaCompatible(child.schema, dest.schema)) return invalidLensProof("hoist schema incompatible"); if (op.removeFromHost !== false) { const hostNew = getPathInfo(newSchema, hostSeg, true); if (hostNew.exists && hostNew.required && isPathRequired(hostNew.schema, [op.name])) { return invalidLensProof("hoist removed field still required"); } } return Result9.ok(undefined); } case "plunge": { const fromSegRes = parsePointerResult(op.from, "invalid_lens_proof"); if (Result9.isError(fromSegRes)) return fromSegRes; const hostSegRes = parsePointerResult(op.host, "invalid_lens_proof"); if (Result9.isError(hostSegRes)) return hostSegRes; const fromSeg = fromSegRes.value; const hostSeg = hostSegRes.value; const src = getPathInfo(oldSchema, fromSeg, true); if (!src.exists || !src.required) return invalidLensProof("plunge source not required"); if (op.createHost !== false) { const parentRes = ensureParentRequiredResult(oldSchema, hostSeg, true, "invalid_lens_proof"); if (Result9.isError(parentRes)) return parentRes; } else { const host = getPathInfo(oldSchema, hostSeg, true); if (!host.exists || !host.required || !isObjectSchema(host.schema)) return invalidLensProof("plunge host invalid"); } const hostNew = getPathInfo(newSchema, hostSeg, true); if (!hostNew.exists || !isObjectSchema(hostNew.schema)) return invalidLensProof("plunge host missing in new schema"); const child = getPropertySchema(hostNew.schema, op.name, true); if (!child.exists) return invalidLensProof("plunge name missing in new schema"); if (!isSchemaCompatible(src.schema, child.schema)) return invalidLensProof("plunge schema incompatible"); if (op.removeFromSource !== false && isPathRequired(newSchema, fromSeg)) { return invalidLensProof("plunge removed field still required"); } return Result9.ok(undefined); } case "wrap": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; let src = getPathInfo(oldSchema, pathSeg, true); if (!src.exists || !src.required) { const renameRes = findRenameToResult(priorOps, pathSeg); if (Result9.isError(renameRes)) return renameRes; if (renameRes.value) { const fromRes = parsePointerResult(renameRes.value.from, "invalid_lens_proof"); if (Result9.isError(fromRes)) return fromRes; src = getPathInfo(oldSchema, fromRes.value, true); } } if (!src.exists || !src.required) return invalidLensProof("wrap path not required"); const dest = getPathInfo(newSchema, pathSeg, true); if (!dest.exists || !isArraySchema(dest.schema)) return invalidLensProof("wrap dest not array"); const items = getItemsSchema(dest.schema); if (!isSchemaCompatible(src.schema, items)) return invalidLensProof("wrap items incompatible"); return Result9.ok(undefined); } case "head": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; const src = getPathInfo(oldSchema, pathSeg, true); if (!src.exists || !src.required || !isArraySchema(src.schema)) return invalidLensProof("head path not required array"); const hasEnum = src.schema && Array.isArray(src.schema.enum); if (hasEnum) { const ok = src.schema.enum.every((v) => Array.isArray(v) && v.length > 0); if (!ok) return invalidLensProof("head requires enum non-empty"); } const minItems = src.schema?.minItems; if (!hasEnum && (minItems === undefined || minItems < 1)) { return invalidLensProof("head requires minItems"); } if (minItems !== undefined && minItems < 1) return invalidLensProof("head requires non-empty array"); const dest = getPathInfo(newSchema, pathSeg, true); if (!dest.exists) return invalidLensProof("head dest missing"); const items = getItemsSchema(src.schema); if (!isSchemaCompatible(items, dest.schema)) return invalidLensProof("head schema incompatible"); return Result9.ok(undefined); } case "convert": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; const src = getPathInfo(oldSchema, pathSeg, true); if (!src.exists || !src.required) return invalidLensProof("convert path not required"); if (op.forward.map && !op.forward.default) { const enumRes = ensureEnumOrConstResult("invalid_lens_proof", src.schema); if (Result9.isError(enumRes)) return enumRes; } const builtin = op.forward.builtin; if (builtin === "string_to_int" || builtin === "rfc3339_to_unix_millis") { const enumRes = ensureEnumOrConstResult("invalid_lens_proof", src.schema); if (Result9.isError(enumRes)) return enumRes; } const dest = getPathInfo(newSchema, pathSeg, true); if (!dest.exists) return invalidLensProof("convert dest missing"); const outType = { type: op.toType }; if (!isSchemaCompatible(outType, dest.schema)) return invalidLensProof("convert dest incompatible"); return Result9.ok(undefined); } case "in": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; const src = getPathInfo(oldSchema, pathSeg, true); const dest = getPathInfo(newSchema, pathSeg, true); if (!src.exists || !src.required || !isObjectSchema(src.schema)) return invalidLensProof("in path invalid"); if (!dest.exists || !isObjectSchema(dest.schema)) return invalidLensProof("in dest invalid"); return validateOpsResult(src.schema, dest.schema, op.ops); } case "map": { const pathSegRes = parsePointerResult(op.path, "invalid_lens_proof"); if (Result9.isError(pathSegRes)) return pathSegRes; const pathSeg = pathSegRes.value; const src = getPathInfo(oldSchema, pathSeg, true); const dest = getPathInfo(newSchema, pathSeg, true); if (!src.exists || !src.required || !isArraySchema(src.schema)) return invalidLensProof("map path invalid"); if (!dest.exists || !isArraySchema(dest.schema)) return invalidLensProof("map dest invalid"); const srcItems = getItemsSchema(src.schema); const destItems = getItemsSchema(dest.schema); return validateOpsResult(srcItems, destItems, op.ops); } default: return invalidLensProof(`unknown op: ${op.op}`); } } function validateOpsResult(oldSchema, newSchema, ops) { for (let i = 0;i < ops.length; i++) { const res = validateOpResult(oldSchema, newSchema, ops[i], ops.slice(0, i), ops.slice(i + 1)); if (Result9.isError(res)) return res; } return Result9.ok(undefined); } function validateLensAgainstSchemasResult(oldSchema, newSchema, lens) { return validateOpsResult(oldSchema, newSchema, lens.ops); } function fillOpsDefaultsResult(ops, newSchema) { for (const op of ops) { if (op.op === "add") { if (op.default === undefined) { const pathRes = parsePointerResult(op.path, "invalid_lens_defaults"); if (Result9.isError(pathRes)) return pathRes; const info = getPathInfo(newSchema, pathRes.value, true); if (!info.exists) return invalidLensDefaults("add dest missing in new schema"); const def = deriveDefault(info.schema); if (def === undefined) return invalidLensDefaults("add missing default"); op.default = def; } continue; } if (op.op === "in") { const pathRes = parsePointerResult(op.path, "invalid_lens_defaults"); if (Result9.isError(pathRes)) return pathRes; const info = getPathInfo(newSchema, pathRes.value, true); if (!info.exists) return invalidLensDefaults("in dest missing in new schema"); const nestedRes = fillOpsDefaultsResult(op.ops, info.schema); if (Result9.isError(nestedRes)) return nestedRes; continue; } if (op.op === "map") { const pathRes = parsePointerResult(op.path, "invalid_lens_defaults"); if (Result9.isError(pathRes)) return pathRes; const info = getPathInfo(newSchema, pathRes.value, true); if (!info.exists) return invalidLensDefaults("map dest missing in new schema"); const items = getItemsSchema(info.schema); const nestedRes = fillOpsDefaultsResult(op.ops, items); if (Result9.isError(nestedRes)) return nestedRes; } } return Result9.ok(undefined); } function cloneLensForDefaultsResult(lens) { try { return Result9.ok(JSON.parse(JSON.stringify(lens))); } catch (e) { return invalidLensDefaults(String(e?.message ?? e)); } } function fillLensDefaultsResult(lens, newSchema) { const copyRes = cloneLensForDefaultsResult(lens); if (Result9.isError(copyRes)) return copyRes; const fillRes = fillOpsDefaultsResult(copyRes.value.ops, newSchema); if (Result9.isError(fillRes)) return fillRes; return Result9.ok(copyRes.value); } // src/touch/spec.ts import { Result as Result10 } from "better-result"; function invalidInterpreter(message) { return Result10.err({ kind: "invalid_interpreter", message }); } function parseNumberField(value, defaultValue, message, predicate) { const n = value === undefined ? defaultValue : Number(value); if (!Number.isFinite(n) || !predicate(n)) return invalidInterpreter(message); return Result10.ok(n); } function parseIntegerField(value, defaultValue, message, predicate) { const n = value === undefined ? defaultValue : Number(value); if (!Number.isFinite(n) || !Number.isInteger(n) || !predicate(n)) return invalidInterpreter(message); return Result10.ok(n); } function validateTouchConfigResult(raw) { if (!raw || typeof raw !== "object") return invalidInterpreter("interpreter.touch must be an object"); const enabled = !!raw.enabled; if (!enabled) { return Result10.ok({ enabled: false }); } if (raw.storage !== undefined) { return invalidInterpreter("interpreter.touch.storage is no longer supported; touch always uses the in-memory journal"); } if (raw.derivedStream !== undefined) { return invalidInterpreter("interpreter.touch.derivedStream is no longer supported"); } if (raw.retention !== undefined) { return invalidInterpreter("interpreter.touch.retention is no longer supported"); } const coarseIntervalMsRes = parseNumberField(raw.coarseIntervalMs, 100, "interpreter.touch.coarseIntervalMs must be > 0", (n) => n > 0); if (Result10.isError(coarseIntervalMsRes)) return coarseIntervalMsRes; const touchCoalesceWindowMsRes = parseNumberField(raw.touchCoalesceWindowMs, 100, "interpreter.touch.touchCoalesceWindowMs must be > 0", (n) => n > 0); if (Result10.isError(touchCoalesceWindowMsRes)) return touchCoalesceWindowMsRes; const onMissingBefore = raw.onMissingBefore === undefined ? "coarse" : raw.onMissingBefore; if (onMissingBefore !== "coarse" && onMissingBefore !== "skipBefore" && onMissingBefore !== "error") { return invalidInterpreter("interpreter.touch.onMissingBefore must be coarse|skipBefore|error"); } const templates = raw.templates && typeof raw.templates === "object" ? raw.templates : {}; const defaultInactivityTtlMsRes = parseNumberField(templates.defaultInactivityTtlMs, 60 * 60 * 1000, "interpreter.touch.templates.defaultInactivityTtlMs must be >= 0", (n) => n >= 0); if (Result10.isError(defaultInactivityTtlMsRes)) return defaultInactivityTtlMsRes; const lastSeenPersistIntervalMsRes = parseNumberField(templates.lastSeenPersistIntervalMs, 5 * 60 * 1000, "interpreter.touch.templates.lastSeenPersistIntervalMs must be > 0", (n) => n > 0); if (Result10.isError(lastSeenPersistIntervalMsRes)) return lastSeenPersistIntervalMsRes; const gcIntervalMsRes = parseNumberField(templates.gcIntervalMs, 60000, "interpreter.touch.templates.gcIntervalMs must be > 0", (n) => n > 0); if (Result10.isError(gcIntervalMsRes)) return gcIntervalMsRes; const maxActiveTemplatesPerEntityRes = parseNumberField(templates.maxActiveTemplatesPerEntity, 256, "interpreter.touch.templates.maxActiveTemplatesPerEntity must be > 0", (n) => n > 0); if (Result10.isError(maxActiveTemplatesPerEntityRes)) return maxActiveTemplatesPerEntityRes; const maxActiveTemplatesPerStreamRes = parseNumberField(templates.maxActiveTemplatesPerStream, 2048, "interpreter.touch.templates.maxActiveTemplatesPerStream must be > 0", (n) => n > 0); if (Result10.isError(maxActiveTemplatesPerStreamRes)) return maxActiveTemplatesPerStreamRes; const activationRateLimitPerMinuteRes = parseNumberField(templates.activationRateLimitPerMinute, 100, "interpreter.touch.templates.activationRateLimitPerMinute must be >= 0", (n) => n >= 0); if (Result10.isError(activationRateLimitPerMinuteRes)) return activationRateLimitPerMinuteRes; if (raw.metrics !== undefined) { return invalidInterpreter("interpreter.touch.metrics is not supported; live metrics are a global server feature"); } const memoryRaw = raw.memory && typeof raw.memory === "object" ? raw.memory : {}; const bucketMsRes = parseIntegerField(memoryRaw.bucketMs, 100, "interpreter.touch.memory.bucketMs must be an integer > 0", (n) => n > 0); if (Result10.isError(bucketMsRes)) return bucketMsRes; const filterPow2Res = parseIntegerField(memoryRaw.filterPow2, 22, "interpreter.touch.memory.filterPow2 must be an integer in [10,30]", (n) => n >= 10 && n <= 30); if (Result10.isError(filterPow2Res)) return filterPow2Res; const kRes = parseIntegerField(memoryRaw.k, 4, "interpreter.touch.memory.k must be an integer in [1,8]", (n) => n >= 1 && n <= 8); if (Result10.isError(kRes)) return kRes; const pendingMaxKeysRes = parseIntegerField(memoryRaw.pendingMaxKeys, 1e5, "interpreter.touch.memory.pendingMaxKeys must be an integer > 0", (n) => n > 0); if (Result10.isError(pendingMaxKeysRes)) return pendingMaxKeysRes; const keyIndexMaxKeysRes = parseIntegerField(memoryRaw.keyIndexMaxKeys, 32, "interpreter.touch.memory.keyIndexMaxKeys must be an integer in [1,1024]", (n) => n >= 1 && n <= 1024); if (Result10.isError(keyIndexMaxKeysRes)) return keyIndexMaxKeysRes; const hotKeyTtlMsRes = parseIntegerField(memoryRaw.hotKeyTtlMs, 1e4, "interpreter.touch.memory.hotKeyTtlMs must be an integer > 0", (n) => n > 0); if (Result10.isError(hotKeyTtlMsRes)) return hotKeyTtlMsRes; const hotTemplateTtlMsRes = parseIntegerField(memoryRaw.hotTemplateTtlMs, 1e4, "interpreter.touch.memory.hotTemplateTtlMs must be an integer > 0", (n) => n > 0); if (Result10.isError(hotTemplateTtlMsRes)) return hotTemplateTtlMsRes; const hotMaxKeysRes = parseIntegerField(memoryRaw.hotMaxKeys, 1e6, "interpreter.touch.memory.hotMaxKeys must be an integer > 0", (n) => n > 0); if (Result10.isError(hotMaxKeysRes)) return hotMaxKeysRes; const hotMaxTemplatesRes = parseIntegerField(memoryRaw.hotMaxTemplates, 4096, "interpreter.touch.memory.hotMaxTemplates must be an integer > 0", (n) => n > 0); if (Result10.isError(hotMaxTemplatesRes)) return hotMaxTemplatesRes; const lagDegradeFineTouchesAtSourceOffsetsRes = parseIntegerField(raw.lagDegradeFineTouchesAtSourceOffsets, 5000, "interpreter.touch.lagDegradeFineTouchesAtSourceOffsets must be an integer >= 0", (n) => n >= 0); if (Result10.isError(lagDegradeFineTouchesAtSourceOffsetsRes)) return lagDegradeFineTouchesAtSourceOffsetsRes; const lagRecoverFineTouchesAtSourceOffsetsRes = parseIntegerField(raw.lagRecoverFineTouchesAtSourceOffsets, 1000, "interpreter.touch.lagRecoverFineTouchesAtSourceOffsets must be an integer >= 0", (n) => n >= 0); if (Result10.isError(lagRecoverFineTouchesAtSourceOffsetsRes)) return lagRecoverFineTouchesAtSourceOffsetsRes; const fineTouchBudgetPerBatchRes = parseIntegerField(raw.fineTouchBudgetPerBatch, 2000, "interpreter.touch.fineTouchBudgetPerBatch must be an integer >= 0", (n) => n >= 0); if (Result10.isError(fineTouchBudgetPerBatchRes)) return fineTouchBudgetPerBatchRes; const fineTokensPerSecondRes = parseIntegerField(raw.fineTokensPerSecond, 200000, "interpreter.touch.fineTokensPerSecond must be an integer >= 0", (n) => n >= 0); if (Result10.isError(fineTokensPerSecondRes)) return fineTokensPerSecondRes; const fineBurstTokensRes = parseIntegerField(raw.fineBurstTokens, 400000, "interpreter.touch.fineBurstTokens must be an integer >= 0", (n) => n >= 0); if (Result10.isError(fineBurstTokensRes)) return fineBurstTokensRes; const lagReservedFineTouchBudgetPerBatchRes = parseIntegerField(raw.lagReservedFineTouchBudgetPerBatch, 200, "interpreter.touch.lagReservedFineTouchBudgetPerBatch must be an integer >= 0", (n) => n >= 0); if (Result10.isError(lagReservedFineTouchBudgetPerBatchRes)) return lagReservedFineTouchBudgetPerBatchRes; return Result10.ok({ enabled: true, coarseIntervalMs: coarseIntervalMsRes.value, touchCoalesceWindowMs: touchCoalesceWindowMsRes.value, onMissingBefore, lagDegradeFineTouchesAtSourceOffsets: lagDegradeFineTouchesAtSourceOffsetsRes.value, lagRecoverFineTouchesAtSourceOffsets: lagRecoverFineTouchesAtSourceOffsetsRes.value, fineTouchBudgetPerBatch: fineTouchBudgetPerBatchRes.value, fineTokensPerSecond: fineTokensPerSecondRes.value, fineBurstTokens: fineBurstTokensRes.value, lagReservedFineTouchBudgetPerBatch: lagReservedFineTouchBudgetPerBatchRes.value, memory: { bucketMs: bucketMsRes.value, filterPow2: filterPow2Res.value, k: kRes.value, pendingMaxKeys: pendingMaxKeysRes.value, keyIndexMaxKeys: keyIndexMaxKeysRes.value, hotKeyTtlMs: hotKeyTtlMsRes.value, hotTemplateTtlMs: hotTemplateTtlMsRes.value, hotMaxKeys: hotMaxKeysRes.value, hotMaxTemplates: hotMaxTemplatesRes.value }, templates: { defaultInactivityTtlMs: defaultInactivityTtlMsRes.value, lastSeenPersistIntervalMs: lastSeenPersistIntervalMsRes.value, gcIntervalMs: gcIntervalMsRes.value, maxActiveTemplatesPerEntity: maxActiveTemplatesPerEntityRes.value, maxActiveTemplatesPerStream: maxActiveTemplatesPerStreamRes.value, activationRateLimitPerMinute: activationRateLimitPerMinuteRes.value } }); } function validateStreamInterpreterConfigResult(raw) { if (!raw || typeof raw !== "object") return invalidInterpreter("interpreter must be an object"); if (raw.apiVersion !== "durable.streams/stream-interpreter/v1") { return invalidInterpreter("invalid interpreter apiVersion"); } const formatRaw = raw.format === undefined ? undefined : raw.format; if (formatRaw !== undefined && formatRaw !== "durable.streams/state-protocol/v1") { return invalidInterpreter("interpreter.format must be durable.streams/state-protocol/v1"); } if (raw.variants !== undefined) { return invalidInterpreter("interpreter.variants is not supported (State Protocol is the only supported format)"); } let touch; if (raw.touch !== undefined) { const touchRes = validateTouchConfigResult(raw.touch); if (Result10.isError(touchRes)) return invalidInterpreter(touchRes.error.message); touch = touchRes.value; } return Result10.ok({ apiVersion: "durable.streams/stream-interpreter/v1", format: "durable.streams/state-protocol/v1", touch }); } function isTouchEnabled(cfg) { return !!cfg?.touch?.enabled; } // src/schema/registry.ts var AJV = new Ajv({ allErrors: true, strict: false, allowUnionTypes: true, validateSchema: false }); var LENS_VALIDATOR = AJV.compile(DURABLE_LENS_V1_SCHEMA); function sha256Hex(input) { return createHash("sha256").update(input).digest("hex"); } function defaultRegistry(stream) { return { apiVersion: "durable.streams/schema-registry/v1", schema: stream, currentVersion: 0, boundaries: [], schemas: {}, lenses: {} }; } function ensureNoRefResult(schema) { const stack = [schema]; while (stack.length > 0) { const cur = stack.pop(); if (!cur || typeof cur !== "object") continue; if (Object.prototype.hasOwnProperty.call(cur, "$ref")) { return Result11.err({ message: "external $ref is not supported" }); } for (const v of Object.values(cur)) { if (v && typeof v === "object") stack.push(v); } } return Result11.ok(undefined); } function validateJsonSchemaResult(schema) { const noRefRes = ensureNoRefResult(schema); if (Result11.isError(noRefRes)) return noRefRes; try { const validate = AJV.compile(schema); if (!validate) return Result11.err({ message: "schema validation failed" }); } catch (e) { return Result11.err({ message: String(e?.message ?? e) }); } return Result11.ok(undefined); } function parseRegistryResult(stream, json) { let raw; try { raw = JSON.parse(json); } catch (e) { return Result11.err({ message: String(e?.message ?? e) }); } if (!raw || typeof raw !== "object") return Result11.err({ message: "invalid schema registry" }); const reg = raw; if (reg.apiVersion !== "durable.streams/schema-registry/v1") return Result11.err({ message: "invalid registry apiVersion" }); if (!reg.schema) reg.schema = stream; if (!Array.isArray(reg.boundaries)) reg.boundaries = []; if (!reg.schemas || typeof reg.schemas !== "object") reg.schemas = {}; if (!reg.lenses || typeof reg.lenses !== "object") reg.lenses = {}; if (typeof reg.currentVersion !== "number") reg.currentVersion = 0; if (reg.interpreter === null) delete reg.interpreter; return Result11.ok(reg); } function serializeRegistry(reg) { return JSON.stringify(reg); } function validateLensResult(raw) { const ok = LENS_VALIDATOR(raw); if (!ok) { const msg = AJV.errorsText(LENS_VALIDATOR.errors || undefined); return Result11.err({ message: `invalid lens: ${msg}` }); } return Result11.ok(raw); } function bigintToNumberSafeResult(v) { const max = BigInt(Number.MAX_SAFE_INTEGER); if (v > max) return Result11.err({ message: "offset exceeds MAX_SAFE_INTEGER" }); return Result11.ok(Number(v)); } class SchemaRegistryStore { db; registryCache; validatorCache; lensCache; lensChainCache; constructor(db, opts) { this.db = db; this.registryCache = new LruCache(opts?.registryCacheEntries ?? 1024); this.validatorCache = new LruCache(opts?.validatorCacheEntries ?? 256); this.lensCache = new LruCache(opts?.lensCacheEntries ?? 256); this.lensChainCache = new LruCache(opts?.lensCacheEntries ?? 256); } loadRow(stream) { return this.db.getSchemaRegistry(stream); } getRegistry(stream) { const res = this.getRegistryResult(stream); if (Result11.isError(res)) throw dsError(res.error.message, { code: res.error.code }); return res.value; } getRegistryResult(stream) { const row = this.loadRow(stream); if (!row) return Result11.ok(defaultRegistry(stream)); const cached = this.registryCache.get(stream); if (cached && cached.updatedAtMs === row.updated_at_ms) return Result11.ok(cached.reg); const parseRes = parseRegistryResult(stream, row.registry_json); if (Result11.isError(parseRes)) { return Result11.err({ kind: "invalid_registry", message: parseRes.error.message }); } const reg = parseRes.value; this.registryCache.set(stream, { reg, updatedAtMs: row.updated_at_ms }); return Result11.ok(reg); } updateRegistry(stream, streamRow, update) { const res = this.updateRegistryResult(stream, streamRow, update); if (Result11.isError(res)) throw dsError(res.error.message, { code: res.error.code }); return res.value; } updateRegistryResult(stream, streamRow, update) { let validatedInterpreter = undefined; if (update.routingKey) { const pointerRes = parseJsonPointerResult(update.routingKey.jsonPointer); if (Result11.isError(pointerRes)) { return Result11.err({ kind: "bad_request", message: pointerRes.error.message }); } if (typeof update.routingKey.required !== "boolean") { return Result11.err({ kind: "bad_request", message: "routingKey.required must be boolean" }); } } if (update.interpreter !== undefined) { if (update.interpreter === null) { validatedInterpreter = null; } else { const interpreterRes = validateStreamInterpreterConfigResult(update.interpreter); if (Result11.isError(interpreterRes)) { return Result11.err({ kind: "bad_request", message: interpreterRes.error.message }); } validatedInterpreter = interpreterRes.value; } } if (update.schema === undefined) return Result11.err({ kind: "bad_request", message: "missing schema" }); const schemaRes = validateJsonSchemaResult(update.schema); if (Result11.isError(schemaRes)) return Result11.err({ kind: "bad_request", message: schemaRes.error.message }); const regRes = this.getRegistryResult(stream); if (Result11.isError(regRes)) return Result11.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code }); const reg = regRes.value; const currentVersion = reg.currentVersion ?? 0; const streamEmpty = streamRow.next_offset === 0n; if (currentVersion === 0) { if (!streamEmpty) return Result11.err({ kind: "bad_request", message: "first schema requires empty stream" }); if (update.lens) { const lensRes2 = validateLensResult(update.lens); if (Result11.isError(lensRes2)) return Result11.err({ kind: "bad_request", message: lensRes2.error.message }); if (lensRes2.value.from !== 0 || lensRes2.value.to !== 1) { return Result11.err({ kind: "version_mismatch", message: "lens version mismatch", code: "schema_lens_version_mismatch" }); } } const nextReg2 = { apiVersion: "durable.streams/schema-registry/v1", schema: stream, currentVersion: 1, routingKey: update.routingKey, interpreter: update.interpreter === undefined ? reg.interpreter : validatedInterpreter ?? undefined, boundaries: [{ offset: 0, version: 1 }], schemas: { ...reg.schemas, ["1"]: update.schema }, lenses: { ...reg.lenses } }; this.persist(stream, nextReg2); this.syncInterpreterState(stream, nextReg2); return Result11.ok(nextReg2); } if (!update.lens) return Result11.err({ kind: "bad_request", message: "lens required" }); const lensRes = validateLensResult(update.lens); if (Result11.isError(lensRes)) return Result11.err({ kind: "bad_request", message: lensRes.error.message }); const lens = lensRes.value; if (lens.from !== currentVersion || lens.to !== currentVersion + 1) { return Result11.err({ kind: "version_mismatch", message: "lens version mismatch", code: "schema_lens_version_mismatch" }); } if (lens.schema && lens.schema !== reg.schema) return Result11.err({ kind: "bad_request", message: "lens schema mismatch" }); const oldSchema = reg.schemas[String(currentVersion)]; if (!oldSchema) return Result11.err({ kind: "bad_request", message: "missing current schema" }); const proofRes = validateLensAgainstSchemasResult(oldSchema, update.schema, lens); if (Result11.isError(proofRes)) return Result11.err({ kind: "bad_request", message: proofRes.error.message }); const defaultsRes = fillLensDefaultsResult(lens, update.schema); if (Result11.isError(defaultsRes)) return Result11.err({ kind: "bad_request", message: defaultsRes.error.message }); const boundaryRes = bigintToNumberSafeResult(streamRow.next_offset); if (Result11.isError(boundaryRes)) return Result11.err({ kind: "bad_request", message: boundaryRes.error.message }); const nextVersion = currentVersion + 1; const nextReg = { apiVersion: "durable.streams/schema-registry/v1", schema: reg.schema ?? stream, currentVersion: nextVersion, routingKey: update.routingKey ?? reg.routingKey, interpreter: update.interpreter === undefined ? reg.interpreter : validatedInterpreter ?? undefined, boundaries: [...reg.boundaries, { offset: boundaryRes.value, version: nextVersion }], schemas: { ...reg.schemas, [String(nextVersion)]: update.schema }, lenses: { ...reg.lenses, [String(currentVersion)]: defaultsRes.value } }; this.persist(stream, nextReg); this.syncInterpreterState(stream, nextReg); return Result11.ok(nextReg); } updateRoutingKey(stream, routingKey) { const res = this.updateRoutingKeyResult(stream, routingKey); if (Result11.isError(res)) throw dsError(res.error.message, { code: res.error.code }); return res.value; } updateRoutingKeyResult(stream, routingKey) { if (routingKey) { const pointerRes = parseJsonPointerResult(routingKey.jsonPointer); if (Result11.isError(pointerRes)) { return Result11.err({ kind: "bad_request", message: pointerRes.error.message }); } if (typeof routingKey.required !== "boolean") { return Result11.err({ kind: "bad_request", message: "routingKey.required must be boolean" }); } } const regRes = this.getRegistryResult(stream); if (Result11.isError(regRes)) return Result11.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code }); const nextReg = { ...regRes.value, routingKey: routingKey ?? undefined }; this.persist(stream, nextReg); this.syncInterpreterState(stream, nextReg); return Result11.ok(nextReg); } updateInterpreter(stream, interpreter) { const res = this.updateInterpreterResult(stream, interpreter); if (Result11.isError(res)) throw dsError(res.error.message, { code: res.error.code }); return res.value; } updateInterpreterResult(stream, interpreter) { let validatedInterpreter = interpreter; if (interpreter) { const interpreterRes = validateStreamInterpreterConfigResult(interpreter); if (Result11.isError(interpreterRes)) { return Result11.err({ kind: "bad_request", message: interpreterRes.error.message }); } validatedInterpreter = interpreterRes.value; } const regRes = this.getRegistryResult(stream); if (Result11.isError(regRes)) return Result11.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code }); const nextReg = { ...regRes.value, interpreter: validatedInterpreter ?? undefined }; this.persist(stream, nextReg); this.syncInterpreterState(stream, nextReg); return Result11.ok(nextReg); } persist(stream, reg) { const json = serializeRegistry(reg); this.db.upsertSchemaRegistry(stream, json); this.registryCache.set(stream, { reg, updatedAtMs: this.db.nowMs() }); } syncInterpreterState(stream, reg) { if (isTouchEnabled(reg.interpreter)) { this.db.ensureStreamInterpreter(stream); } else { this.db.deleteStreamInterpreter(stream); } } getValidatorForVersion(reg, version) { const schema = reg.schemas[String(version)]; if (!schema) return null; const hash = sha256Hex(JSON.stringify(schema)); const cached = this.validatorCache.get(hash); if (cached) return cached; const validate = AJV.compile(schema); this.validatorCache.set(hash, validate); return validate; } getLensChain(reg, fromVersion, toVersion) { const res = this.getLensChainResult(reg, fromVersion, toVersion); if (Result11.isError(res)) throw dsError(res.error.message, { code: res.error.code }); return res.value; } getLensChainResult(reg, fromVersion, toVersion) { const key = `${reg.schema}:${fromVersion}->${toVersion}`; const cached = this.lensChainCache.get(key); if (cached) return Result11.ok(cached); const chain = []; for (let v = fromVersion;v < toVersion; v++) { const lensRaw = reg.lenses[String(v)]; if (!lensRaw) { return Result11.err({ kind: "invalid_lens_chain", message: `missing lens v${v}->v${v + 1}` }); } const hash = sha256Hex(JSON.stringify(lensRaw)); let compiled = this.lensCache.get(hash); if (!compiled) { const compiledRes = compileLensResult(lensFromJson(lensRaw)); if (Result11.isError(compiledRes)) { return Result11.err({ kind: "invalid_lens_chain", message: compiledRes.error.message }); } compiled = compiledRes.value; this.lensCache.set(hash, compiled); } chain.push(compiled); } this.lensChainCache.set(key, chain); return Result11.ok(chain); } } // src/expiry_sweeper.ts class ExpirySweeper { cfg; db; timer = null; running = false; constructor(cfg, db) { this.cfg = cfg; this.db = db; } start() { if (this.timer || this.cfg.expirySweepIntervalMs <= 0) return; this.timer = setInterval(() => { this.tick(); }, this.cfg.expirySweepIntervalMs); } stop() { if (this.timer) clearInterval(this.timer); this.timer = null; } async tick() { if (this.running) return; this.running = true; try { const expired = this.db.listExpiredStreams(this.cfg.expirySweepBatchLimit); if (expired.length === 0) return; for (const stream of expired) { try { this.db.deleteStream(stream); } catch {} } } finally { this.running = false; } } } // src/backpressure.ts class BackpressureGate { maxBytes; currentBytes; reservedBytes; constructor(maxBytes, initialBytes) { this.maxBytes = maxBytes; this.currentBytes = Math.max(0, initialBytes); this.reservedBytes = 0; } enabled() { return this.maxBytes > 0; } reserve(bytes) { if (this.maxBytes <= 0) return true; if (bytes <= 0) return true; if (this.currentBytes + this.reservedBytes + bytes > this.maxBytes) return false; this.reservedBytes += bytes; return true; } commit(bytes, reservedBytes = bytes) { if (this.maxBytes <= 0) return; if (bytes <= 0) return; if (reservedBytes > 0) this.reservedBytes = Math.max(0, this.reservedBytes - reservedBytes); this.currentBytes += bytes; } release(bytes) { if (this.maxBytes <= 0) return; if (bytes <= 0) return; this.reservedBytes = Math.max(0, this.reservedBytes - bytes); } adjustOnSeal(payloadBytes, segmentBytes) { if (this.maxBytes <= 0) return; const delta = segmentBytes - payloadBytes; this.currentBytes = Math.max(0, this.currentBytes + delta); } adjustOnUpload(segmentBytes) { if (this.maxBytes <= 0) return; this.currentBytes = Math.max(0, this.currentBytes - segmentBytes); } adjustOnWalTrim(payloadBytes) { if (this.maxBytes <= 0) return; if (payloadBytes <= 0) return; this.currentBytes = Math.max(0, this.currentBytes - payloadBytes); } getCurrentBytes() { return this.currentBytes; } getMaxBytes() { return this.maxBytes; } isOverLimit() { if (this.maxBytes <= 0) return false; return this.currentBytes + this.reservedBytes >= this.maxBytes; } } // src/memory.ts class MemoryGuard { limitBytes; resumeBytes; intervalMs; onSample; heapSnapshotPath; heapSnapshotMinIntervalMs; timer = null; overLimit = false; maxRssBytes = 0; lastRssBytes = 0; lastGcMs = 0; lastSnapshotMs = 0; constructor(limitBytes, opts = {}) { this.limitBytes = Math.max(0, limitBytes); const resumeFraction = Math.min(1, Math.max(0.5, opts.resumeFraction ?? 1)); this.resumeBytes = Math.floor(this.limitBytes * resumeFraction); this.intervalMs = Math.max(50, opts.intervalMs ?? 1000); this.onSample = opts.onSample; this.heapSnapshotPath = opts.heapSnapshotPath; this.heapSnapshotMinIntervalMs = Math.max(1000, opts.heapSnapshotMinIntervalMs ?? 60000); } start() { if (this.timer) return; this.sample(); this.timer = setInterval(() => this.sample(), this.intervalMs); } stop() { if (this.timer) clearInterval(this.timer); this.timer = null; } sample() { const rss = process.memoryUsage().rss; this.lastRssBytes = rss; if (rss > this.maxRssBytes) this.maxRssBytes = rss; if (this.onSample) { const overLimit = this.limitBytes > 0 && rss > this.limitBytes; try { this.onSample(rss, overLimit, this.limitBytes); } catch {} } if (this.limitBytes <= 0) return; if (this.overLimit) { if (rss <= this.resumeBytes) this.overLimit = false; } else if (rss > this.limitBytes) { this.overLimit = true; } } shouldAllow() { if (this.limitBytes <= 0) return true; return !this.overLimit; } isOverLimit() { return this.overLimit; } getMaxRssBytes() { return this.maxRssBytes; } snapshotMaxRssBytes(reset = true) { const max = this.maxRssBytes; if (reset) this.maxRssBytes = this.lastRssBytes; return max; } getLastRssBytes() { return this.lastRssBytes; } getLimitBytes() { return this.limitBytes; } maybeGc(reason) { const gcFn = globalThis?.Bun?.gc; if (typeof gcFn !== "function") return; const now = Date.now(); if (now - this.lastGcMs < 1e4) return; this.lastGcMs = now; const before = process.memoryUsage().rss; try { gcFn(true); } catch { try { gcFn(); } catch { return; } } const after = process.memoryUsage().rss; console.warn(`[gc] forced GC (${reason}) rss ${formatBytes2(before)} -> ${formatBytes2(after)}`); } maybeHeapSnapshot(reason) { if (!this.heapSnapshotPath) return; const now = Date.now(); if (now - this.lastSnapshotMs < this.heapSnapshotMinIntervalMs) return; this.lastSnapshotMs = now; this.writeHeapSnapshot(reason); } async writeHeapSnapshot(reason) { try { const v8 = await import("v8"); if (typeof v8.writeHeapSnapshot !== "function") return; const fs = await import("node:fs"); try { fs.unlinkSync(this.heapSnapshotPath); } catch {} const before = process.memoryUsage().rss; v8.writeHeapSnapshot(this.heapSnapshotPath); const after = process.memoryUsage().rss; console.warn(`[heap] snapshot (${reason}) rss ${formatBytes2(before)} -> ${formatBytes2(after)} path=${this.heapSnapshotPath}`); } catch (err) { console.warn(`[heap] snapshot failed (${reason}): ${String(err)}`); } } } function formatBytes2(bytes) { const units = ["b", "kb", "mb", "gb"]; let value = bytes; let idx = 0; while (value >= 1024 && idx < units.length - 1) { value /= 1024; idx += 1; } const digits = idx === 0 ? 0 : 1; return `${value.toFixed(digits)}${units[idx]}`; } // src/touch/worker_pool.ts import { existsSync as existsSync2 } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { Worker } from "node:worker_threads"; import { Result as Result12 } from "better-result"; class TouchInterpreterWorkerPool { cfg; workerCount; workers = []; started = false; generation = 0; nextId = 1; pending = new Map; queue = []; constructor(cfg, workerCount) { this.cfg = cfg; this.workerCount = Math.max(0, Math.floor(workerCount)); } start() { if (this.started) return; this.started = true; this.generation += 1; const generation = this.generation; for (let i = 0;i < this.workerCount; i++) this.spawnWorker(i, generation); } stop() { if (!this.started) return; this.started = false; this.generation += 1; for (const w of this.workers) { try { w.worker.postMessage({ type: "stop" }); } catch {} w.worker.terminate(); } this.workers.length = 0; this.queue.length = 0; for (const [id, p] of this.pending.entries()) { p.resolve(Result12.err({ kind: "worker_pool_failure", message: "worker pool stopped" })); this.pending.delete(id); } } restart() { this.stop(); this.start(); } async processResult(req) { if (!this.started) { return Result12.err({ kind: "worker_pool_unavailable", message: "worker pool not started" }); } if (this.workerCount === 0) { return Result12.err({ kind: "worker_pool_unavailable", message: "worker pool disabled" }); } const id = this.nextId++; const queued = { ...req, id }; const value = await new Promise((resolve2) => { this.pending.set(id, { resolve: resolve2 }); this.queue.push(queued); this.pump(); }); return value; } async process(req) { const res = await this.processResult(req); if (Result12.isError(res)) throw dsError(res.error.message); return res.value; } pump() { if (!this.started) return; if (this.queue.length === 0) return; const slot = this.workers.find((w) => !w.busy); if (!slot) return; const next = this.queue.shift(); if (!next) return; slot.busy = true; slot.currentId = next.id; slot.worker.postMessage({ type: "process", id: next.id, stream: next.stream, fromOffset: next.fromOffset, toOffset: next.toOffset, interpreter: next.interpreter, maxRows: next.maxRows, maxBytes: next.maxBytes, emitFineTouches: next.emitFineTouches, fineTouchBudget: next.fineTouchBudget, fineGranularity: next.fineGranularity, interpretMode: next.interpretMode, filterHotTemplates: next.filterHotTemplates, hotTemplateIds: next.hotTemplateIds }); } spawnWorker(idx, generation = this.generation) { const workerUrl = new URL("../touch/interpreter_worker.js", import.meta.url); let workerSpec = fileURLToPath(workerUrl); if (!existsSync2(workerSpec)) { const fallback = resolve(process.cwd(), "src/touch/interpreter_worker.ts"); if (existsSync2(fallback)) workerSpec = fallback; } const worker = new Worker(workerSpec, { workerData: { config: this.cfg, hostRuntime: detectHostRuntime() }, type: "module", smol: true }); const slot = { worker, busy: false, currentId: null }; this.workers.push(slot); worker.on("message", (msg) => { if (generation !== this.generation) return; if (!msg || typeof msg !== "object") return; if (msg.type === "result") { const p = this.pending.get(msg.id); if (p) { this.pending.delete(msg.id); slot.busy = false; slot.currentId = null; p.resolve(Result12.ok(msg)); } this.pump(); return; } if (msg.type === "error") { const p = this.pending.get(msg.id); if (p) { this.pending.delete(msg.id); slot.busy = false; slot.currentId = null; p.resolve(Result12.err({ kind: "worker_pool_failure", message: msg.message })); } this.pump(); } }); worker.on("error", (err) => { if (generation !== this.generation) return; console.error(`touch interpreter worker ${idx} error`, err); }); worker.on("exit", (code) => { if (generation !== this.generation || !this.started) return; console.error(`touch interpreter worker ${idx} exited with code ${code}, respawning`); if (slot.currentId != null) { const p = this.pending.get(slot.currentId); if (p) { this.pending.delete(slot.currentId); p.resolve(Result12.err({ kind: "worker_pool_failure", message: "worker exited" })); } } slot.busy = false; slot.currentId = null; try { const widx = this.workers.indexOf(slot); if (widx >= 0) this.workers.splice(widx, 1); } catch {} this.spawnWorker(idx, generation); this.pump(); }); } } // src/runtime/hash.ts import { Result as Result13 } from "better-result"; import { createRequire as createRequire3 } from "node:module"; import { fileURLToPath as fileURLToPath2 } from "node:url"; var xxh3Hasher = null; var xxh64Hasher = null; var xxh32Hasher = null; var isBunRuntime = typeof globalThis.Bun !== "undefined"; var require3 = createRequire3(import.meta.url); function loadVendoredModule(name) { const path = fileURLToPath2(new URL(`./hash_vendor/${name}`, import.meta.url)); return require3(path); } if (!isBunRuntime) { const xxh3Module = loadVendoredModule("xxhash3.umd.min.cjs"); const xxh64Module = loadVendoredModule("xxhash64.umd.min.cjs"); const xxh32Module = loadVendoredModule("xxhash32.umd.min.cjs"); xxh3Hasher = await xxh3Module.createXXHash3(); xxh64Hasher = await xxh64Module.createXXHash64(); xxh32Hasher = await xxh32Module.createXXHash32(); } function toBigIntDigest(value) { if (typeof value === "bigint") return value; if (typeof value === "number") return BigInt(value >>> 0); const hex = value.startsWith("0x") ? value.slice(2) : value; if (hex.length === 0) return 0n; return BigInt(`0x${hex}`); } function toHex16(value) { const masked = value & 0xffff_ffff_ffff_ffffn; return masked.toString(16).padStart(16, "0"); } function bunHash64(input, fn) { return fn(input); } function nodeHash64Result(input, hasher, label) { if (!hasher) return Result13.err({ kind: "hasher_not_initialized", message: `${label} hasher not initialized` }); hasher.init(); hasher.update(input); const digest = hasher.digest("hex"); return Result13.ok(toBigIntDigest(digest)); } function nodeHash32Result(input) { if (!xxh32Hasher) return Result13.err({ kind: "hasher_not_initialized", message: "xxh32 hasher not initialized" }); xxh32Hasher.init(); xxh32Hasher.update(input); const digest = xxh32Hasher.digest("hex"); if (typeof digest === "number") return Result13.ok(digest >>> 0); const asBigInt = toBigIntDigest(digest); return Result13.ok(Number(asBigInt & 0xffff_ffffn) >>> 0); } function xxh3BigIntResult(input) { if (isBunRuntime) return Result13.ok(bunHash64(input, (x) => Bun.hash.xxHash3(x))); return nodeHash64Result(input, xxh3Hasher, "xxh3"); } function xxh3HexResult(input) { const res = xxh3BigIntResult(input); if (Result13.isError(res)) return res; return Result13.ok(toHex16(res.value)); } function xxh32Result(input) { if (isBunRuntime) return Result13.ok(Bun.hash.xxHash32(input) >>> 0); return nodeHash32Result(input); } function xxh3BigInt(input) { const res = xxh3BigIntResult(input); if (Result13.isError(res)) throw dsError(res.error.message); return res.value; } function xxh3Hex(input) { const res = xxh3HexResult(input); if (Result13.isError(res)) throw dsError(res.error.message); return res.value; } // src/touch/live_keys.ts function utf8(s) { return new TextEncoder().encode(s); } function encodeU64Be(v) { const out = new Uint8Array(8); let x = v; for (let i = 7;i >= 0; i--) { out[i] = Number(x & 0xffn); x >>= 8n; } return out; } function xxh3Low32(bytes) { const h = xxh3BigInt(bytes); return Number(h & 0xffffffffn) >>> 0; } function canonicalizeTemplateFields(fields) { const out = [...fields].map((f) => ({ name: f.name, encoding: f.encoding })); out.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0); return out; } function templateIdFor(entity, fieldNamesSorted) { const parts = [utf8("tpl\x00"), utf8(entity), utf8("\x00")]; for (let i = 0;i < fieldNamesSorted.length; i++) { if (i > 0) parts.push(utf8("\x00")); parts.push(utf8(fieldNamesSorted[i])); } return xxh3Hex(concat(parts)); } function tableKeyIdFor(entity) { return xxh3Low32(concat([utf8("tbl\x00"), utf8(entity)])); } function templateKeyIdFor(templateIdHex16) { const tplBytes = encodeU64Be(BigInt(`0x${templateIdHex16}`)); return xxh3Low32(concat([utf8("tpl\x00"), tplBytes])); } function concat(parts) { let total = 0; for (const p of parts) total += p.byteLength; const out = new Uint8Array(total); let off = 0; for (const p of parts) { out.set(p, off); off += p.byteLength; } return out; } // src/touch/live_templates.ts function nowIso(ms) { return new Date(ms).toISOString(); } function parseTemplateRow(row) { return { stream: String(row.stream), template_id: String(row.template_id), entity: String(row.entity), fields_json: String(row.fields_json), encodings_json: String(row.encodings_json), state: String(row.state), created_at_ms: typeof row.created_at_ms === "bigint" ? row.created_at_ms : BigInt(row.created_at_ms), last_seen_at_ms: typeof row.last_seen_at_ms === "bigint" ? row.last_seen_at_ms : BigInt(row.last_seen_at_ms), inactivity_ttl_ms: typeof row.inactivity_ttl_ms === "bigint" ? row.inactivity_ttl_ms : BigInt(row.inactivity_ttl_ms), active_from_source_offset: typeof row.active_from_source_offset === "bigint" ? row.active_from_source_offset : BigInt(row.active_from_source_offset), retired_at_ms: row.retired_at_ms == null ? null : typeof row.retired_at_ms === "bigint" ? row.retired_at_ms : BigInt(row.retired_at_ms), retired_reason: row.retired_reason == null ? null : String(row.retired_reason) }; } class LiveTemplateRegistry { db; lastSeenMem = new Map; dirtyLastSeen = new Set; rate = new Map; constructor(db) { this.db = db; } key(stream, templateId) { return `${stream} ${templateId}`; } allowActivation(stream, nowMs, limitPerMinute) { if (limitPerMinute <= 0) return true; const ratePerMs = limitPerMinute / 60000; const st = this.rate.get(stream) ?? { tokens: limitPerMinute, lastRefillMs: nowMs }; const elapsed = Math.max(0, nowMs - st.lastRefillMs); st.tokens = Math.min(limitPerMinute, st.tokens + elapsed * ratePerMs); st.lastRefillMs = nowMs; if (st.tokens < 1) { this.rate.set(stream, st); return false; } st.tokens -= 1; this.rate.set(stream, st); return true; } getActiveTemplateCount(stream) { try { const row = this.db.db.query(`SELECT COUNT(*) as cnt FROM live_templates WHERE stream=? AND state='active';`).get(stream); return Number(row?.cnt ?? 0); } catch { return 0; } } listActiveTemplates(stream) { try { const rows = this.db.db.query(`SELECT template_id, entity, fields_json, encodings_json, last_seen_at_ms FROM live_templates WHERE stream=? AND state='active' ORDER BY entity ASC, template_id ASC;`).all(stream); const out = []; for (const row of rows) { const templateId = String(row.template_id); const entity = String(row.entity); const fields = JSON.parse(String(row.fields_json)); const encodings = JSON.parse(String(row.encodings_json)); if (!Array.isArray(fields) || !Array.isArray(encodings) || fields.length !== encodings.length) continue; const lastSeenAtMs = Number(row.last_seen_at_ms); out.push({ templateId, entity, fields: fields.map(String), encodings: encodings.map(String), lastSeenAtMs }); } return out; } catch { return []; } } activate(args) { const { stream, templates, inactivityTtlMs, nowMs } = args; const { maxActiveTemplatesPerStream, maxActiveTemplatesPerEntity, activationRateLimitPerMinute } = args.limits; const activated = []; const denied = []; const lifecycle = []; const protectedIds = new Set; for (const t of templates) { const entity = typeof t?.entity === "string" ? t.entity.trim() : ""; if (entity === "") { denied.push({ templateId: "0000000000000000", reason: "invalid" }); continue; } if (!Array.isArray(t.fields) || t.fields.length === 0 || t.fields.length > 3) { denied.push({ templateId: "0000000000000000", reason: "invalid" }); continue; } const rawFields = []; for (const f of t.fields) { const name = typeof f?.name === "string" ? String(f.name).trim() : ""; const encoding = f?.encoding; if (name === "") continue; if (encoding !== "string" && encoding !== "int64" && encoding !== "bool" && encoding !== "datetime" && encoding !== "bytes") continue; rawFields.push({ name, encoding }); } if (rawFields.length !== t.fields.length) { denied.push({ templateId: "0000000000000000", reason: "invalid" }); continue; } { const seen = new Set; let ok = true; for (const f of rawFields) { if (seen.has(f.name)) ok = false; seen.add(f.name); } if (!ok) { denied.push({ templateId: "0000000000000000", reason: "invalid" }); continue; } } const fields = canonicalizeTemplateFields(rawFields); const fieldNames = fields.map((f) => f.name); const encodings = fields.map((f) => f.encoding); const templateId = templateIdFor(entity, fieldNames); const existing = this.db.db.query(`SELECT stream, template_id, entity, fields_json, encodings_json, state, created_at_ms, last_seen_at_ms, inactivity_ttl_ms, active_from_source_offset, retired_at_ms, retired_reason FROM live_templates WHERE stream=? AND template_id=? LIMIT 1;`).get(stream, templateId); const alreadyActive = existing && String(existing.state) === "active"; const needsToken = !alreadyActive; if (needsToken && !this.allowActivation(stream, nowMs, activationRateLimitPerMinute)) { denied.push({ templateId, reason: "rate_limited" }); continue; } if (existing) { const row = parseTemplateRow(existing); if (row.entity !== entity) { denied.push({ templateId, reason: "invalid" }); continue; } let storedFields; let storedEnc; try { storedFields = JSON.parse(row.fields_json); storedEnc = JSON.parse(row.encodings_json); } catch { denied.push({ templateId, reason: "invalid" }); continue; } if (!Array.isArray(storedFields) || !Array.isArray(storedEnc) || storedFields.length !== storedEnc.length) { denied.push({ templateId, reason: "invalid" }); continue; } const sf = storedFields.map(String); const se = storedEnc.map(String); if (sf.join("\x00") !== fieldNames.join("\x00")) { denied.push({ templateId, reason: "invalid" }); continue; } if (se.join("\x00") !== encodings.join("\x00")) { denied.push({ templateId, reason: "invalid" }); continue; } if (row.state === "active") { this.db.db.query(`UPDATE live_templates SET last_seen_at_ms=?, inactivity_ttl_ms=? WHERE stream=? AND template_id=?;`).run(nowMs, inactivityTtlMs, stream, templateId); } else { this.db.db.query(`UPDATE live_templates SET state='active', last_seen_at_ms=?, inactivity_ttl_ms=?, active_from_source_offset=?, retired_at_ms=NULL, retired_reason=NULL WHERE stream=? AND template_id=?;`).run(nowMs, inactivityTtlMs, args.baseStreamNextOffset, stream, templateId); } } else { this.db.db.query(`INSERT INTO live_templates( stream, template_id, entity, fields_json, encodings_json, state, created_at_ms, last_seen_at_ms, inactivity_ttl_ms, active_from_source_offset, retired_at_ms, retired_reason ) VALUES(?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, NULL, NULL);`).run(stream, templateId, entity, JSON.stringify(fieldNames), JSON.stringify(encodings), nowMs, nowMs, inactivityTtlMs, args.baseStreamNextOffset); } protectedIds.add(templateId); activated.push({ templateId, state: "active", activeFromTouchOffset: args.activeFromTouchOffset }); lifecycle.push({ type: "live.template_activated", ts: nowIso(nowMs), stream, templateId, entity, fields: fieldNames, encodings, reason: "declared", activeFromTouchOffset: args.activeFromTouchOffset, inactivityTtlMs }); this.markSeen(stream, templateId, nowMs); } const evicted = this.evictToCaps(stream, nowMs, { maxActiveTemplatesPerStream, maxActiveTemplatesPerEntity }, protectedIds); for (const e of evicted) lifecycle.push(e); return { activated, denied, lifecycle }; } heartbeat(stream, templateIdsUsed, nowMs) { for (const id of templateIdsUsed) { const templateId = typeof id === "string" ? id.trim() : ""; if (!/^[0-9a-f]{16}$/.test(templateId)) continue; this.markSeen(stream, templateId, nowMs); } } flushLastSeen(nowMs, persistIntervalMs) { if (this.dirtyLastSeen.size === 0) return; const stmt = this.db.db.query(`UPDATE live_templates SET last_seen_at_ms=? WHERE stream=? AND template_id=? AND state='active';`); try { for (const k of this.dirtyLastSeen) { const item = this.lastSeenMem.get(k); if (!item) { this.dirtyLastSeen.delete(k); continue; } if (nowMs - item.lastPersistMs < persistIntervalMs) continue; const [stream, templateId] = k.split(` `); stmt.run(item.lastSeenMs, stream, templateId); item.lastPersistMs = nowMs; this.dirtyLastSeen.delete(k); } } finally { try { stmt.finalize?.(); } catch {} } } gcRetireExpired(stream, nowMs) { const expired = []; try { const rows = this.db.db.query(`SELECT template_id, entity, fields_json, encodings_json, last_seen_at_ms, inactivity_ttl_ms FROM live_templates WHERE stream=? AND state='active' AND (last_seen_at_ms + inactivity_ttl_ms) < ? ORDER BY last_seen_at_ms ASC LIMIT 1000;`).all(stream, nowMs); expired.push(...rows); } catch { return { retired: [] }; } if (expired.length === 0) return { retired: [] }; const effectiveExpired = []; const refreshLastSeen = this.db.db.query(`UPDATE live_templates SET last_seen_at_ms=? WHERE stream=? AND template_id=? AND state='active';`); try { for (const row of expired) { const templateId = String(row.template_id); const dbLastSeenAtMs = Number(row.last_seen_at_ms); const ttlMs = Number(row.inactivity_ttl_ms); const mem = this.lastSeenMem.get(this.key(stream, templateId)); const memLastSeen = mem ? mem.lastSeenMs : 0; const lastSeenAtMs = Math.max(dbLastSeenAtMs, memLastSeen); if (lastSeenAtMs + ttlMs >= nowMs) { if (mem && memLastSeen > dbLastSeenAtMs) { refreshLastSeen.run(memLastSeen, stream, templateId); mem.lastPersistMs = nowMs; this.dirtyLastSeen.delete(this.key(stream, templateId)); } continue; } effectiveExpired.push(row); } } finally { try { refreshLastSeen.finalize?.(); } catch {} } if (effectiveExpired.length === 0) return { retired: [] }; const retired = []; const update = this.db.db.query(`UPDATE live_templates SET state='retired', retired_reason='inactivity', retired_at_ms=? WHERE stream=? AND template_id=? AND state='active';`); try { for (const row of effectiveExpired) { const templateId = String(row.template_id); const entity = String(row.entity); let fields = []; let encodings = []; try { const f = JSON.parse(String(row.fields_json)); const e = JSON.parse(String(row.encodings_json)); if (Array.isArray(f)) fields = f.map(String); if (Array.isArray(e)) encodings = e.map(String); } catch {} const dbLastSeenAtMs = Number(row.last_seen_at_ms); const mem = this.lastSeenMem.get(this.key(stream, templateId)); const memLastSeen = mem ? mem.lastSeenMs : 0; const lastSeenAtMs = Math.max(dbLastSeenAtMs, memLastSeen); const inactiveForMs = Math.max(0, nowMs - lastSeenAtMs); update.run(nowMs, stream, templateId); retired.push({ type: "live.template_retired", ts: nowIso(nowMs), stream, templateId, entity, fields, encodings, lastSeenAt: nowIso(lastSeenAtMs), inactiveForMs, reason: "inactivity" }); this.lastSeenMem.delete(this.key(stream, templateId)); this.dirtyLastSeen.delete(this.key(stream, templateId)); } } finally { try { update.finalize?.(); } catch {} } return { retired }; } markSeen(stream, templateId, nowMs) { const k = this.key(stream, templateId); const item = this.lastSeenMem.get(k) ?? { lastSeenMs: 0, lastPersistMs: 0 }; if (nowMs > item.lastSeenMs) item.lastSeenMs = nowMs; this.lastSeenMem.set(k, item); this.dirtyLastSeen.add(k); } evictToCaps(stream, nowMs, caps, protectedIds) { const out = []; const { maxActiveTemplatesPerStream, maxActiveTemplatesPerEntity } = caps; let entities = []; try { const rows = this.db.db.query(`SELECT entity, COUNT(*) as cnt FROM live_templates WHERE stream=? AND state='active' GROUP BY entity;`).all(stream); entities = rows.map((r) => ({ entity: String(r.entity), cnt: Number(r.cnt) })); } catch {} for (const e of entities) { if (e.cnt <= maxActiveTemplatesPerEntity) continue; const extra = e.cnt - maxActiveTemplatesPerEntity; const evicted = this.evictLru(stream, nowMs, extra, { entity: e.entity, cap: maxActiveTemplatesPerEntity }, protectedIds); out.push(...evicted); } let activeCount = 0; try { const row = this.db.db.query(`SELECT COUNT(*) as cnt FROM live_templates WHERE stream=? AND state='active';`).get(stream); activeCount = Number(row?.cnt ?? 0); } catch { activeCount = 0; } if (activeCount > maxActiveTemplatesPerStream) { const extra = activeCount - maxActiveTemplatesPerStream; const evicted = this.evictLru(stream, nowMs, extra, { cap: maxActiveTemplatesPerStream }, protectedIds); out.push(...evicted); } return out; } evictLru(stream, nowMs, count, scope, protectedIds) { if (count <= 0) return []; const out = []; const pick = (excludeProtected) => { const params = [stream]; let where = `stream=? AND state='active'`; if (scope.entity) { where += ` AND entity=?`; params.push(scope.entity); } if (excludeProtected && protectedIds.size > 0) { const placeholders = Array.from(protectedIds).map(() => "?").join(", "); where += ` AND template_id NOT IN (${placeholders})`; params.push(...Array.from(protectedIds)); } const q = `SELECT template_id FROM live_templates WHERE ${where} ORDER BY last_seen_at_ms ASC, template_id ASC LIMIT ?;`; params.push(count); try { const rows = this.db.db.query(q).all(...params); return rows.map((r) => String(r.template_id)); } catch { return []; } }; let ids = pick(true); if (ids.length < count) { const extra = pick(false); const merged = []; const seen = new Set; for (const id of [...ids, ...extra]) { if (seen.has(id)) continue; seen.add(id); merged.push(id); if (merged.length >= count) break; } ids = merged; } if (ids.length === 0) return []; const update = this.db.db.query(`UPDATE live_templates SET state='retired', retired_reason='cap_exceeded', retired_at_ms=? WHERE stream=? AND template_id=? AND state='active';`); try { for (const id of ids) { update.run(nowMs, stream, id); out.push({ type: "live.template_evicted", ts: nowIso(nowMs), stream, templateId: id, reason: "cap_exceeded", cap: scope.cap }); this.lastSeenMem.delete(this.key(stream, id)); this.dirtyLastSeen.delete(this.key(stream, id)); } } finally { try { update.finalize?.(); } catch {} } return out; } } // src/touch/live_metrics.ts import { Result as Result14 } from "better-result"; function makeLatencyHistogram() { const bounds = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 1e4, 30000, 120000]; const counts = new Array(bounds.length + 1).fill(0); const record = (ms) => { const x = Math.max(0, Math.floor(ms)); let i = 0; while (i < bounds.length && x > bounds[i]) i++; counts[i] += 1; }; const quantile = (q) => { const total = counts.reduce((a, b) => a + b, 0); if (total === 0) return 0; const target = Math.ceil(total * q); let acc = 0; for (let i = 0;i < counts.length; i++) { acc += counts[i]; if (acc >= target) { return i < bounds.length ? bounds[i] : bounds[bounds.length - 1]; } } return bounds[bounds.length - 1]; }; const reset = () => { for (let i = 0;i < counts.length; i++) counts[i] = 0; }; return { bounds, counts, record, p50: () => quantile(0.5), p95: () => quantile(0.95), p99: () => quantile(0.99), reset }; } function nowIso2(ms) { return new Date(ms).toISOString(); } function envString(name) { const v = process.env[name]; return v && v.trim() !== "" ? v.trim() : null; } function getInstanceId() { return envString("DS_INSTANCE_ID") ?? envString("HOSTNAME") ?? "local"; } function getRegion() { return envString("DS_REGION") ?? "local"; } function defaultCounters(touchCfg) { return { touch: { coarseIntervalMs: touchCfg.coarseIntervalMs ?? 100, coalesceWindowMs: touchCfg.touchCoalesceWindowMs ?? 100, mode: "idle", hotFineKeys: 0, hotTemplates: 0, hotFineKeysActive: 0, hotFineKeysGrace: 0, hotTemplatesActive: 0, hotTemplatesGrace: 0, fineWaitersActive: 0, coarseWaitersActive: 0, broadFineWaitersActive: 0, touchesEmitted: 0, uniqueKeysTouched: 0, tableTouchesEmitted: 0, templateTouchesEmitted: 0, staleResponses: 0, fineTouchesDroppedDueToBudget: 0, fineTouchesSkippedColdTemplate: 0, fineTouchesSkippedColdKey: 0, fineTouchesSkippedTemplateBucket: 0, fineTouchesSuppressedBatchesDueToLag: 0, fineTouchesSuppressedMsDueToLag: 0, fineTouchesSuppressedBatchesDueToBudget: 0 }, gc: { baseWalGcCalls: 0, baseWalGcDeletedRows: 0, baseWalGcDeletedBytes: 0, baseWalGcMsSum: 0, baseWalGcMsMax: 0 }, templates: { activated: 0, retired: 0, evicted: 0, activationDenied: 0 }, wait: { calls: 0, keysWatchedTotal: 0, touched: 0, timeout: 0, stale: 0, latencySumMs: 0, latencyHist: makeLatencyHistogram() }, interpreter: { eventsIn: 0, changesOut: 0, errors: 0, lagSourceOffsets: 0, scannedBatches: 0, scannedButEmitted0Batches: 0, noInterestFastForwardBatches: 0, interpretedThroughDelta: 0, touchesEmittedDelta: 0, commitLagSamples: 0, commitLagMsSum: 0, commitLagHist: makeLatencyHistogram() } }; } class LiveMetricsV2 { db; ingest; metricsStream; enabled; intervalMs; snapshotIntervalMs; snapshotChunkSize; retentionMs; getTouchJournal; timer = null; snapshotTimer = null; retentionTimer = null; lagTimer = null; instanceId = getInstanceId(); region = getRegion(); counters = new Map; lagExpectedMs = 0; lagMaxMs = 0; lagSumMs = 0; lagSamples = 0; constructor(db, ingest, opts) { this.db = db; this.ingest = ingest; this.enabled = opts?.enabled !== false; this.metricsStream = opts?.stream ?? "live.metrics"; this.intervalMs = opts?.intervalMs ?? 1000; this.snapshotIntervalMs = opts?.snapshotIntervalMs ?? 60000; this.snapshotChunkSize = opts?.snapshotChunkSize ?? 200; this.retentionMs = opts?.retentionMs ?? 7 * 24 * 60 * 60 * 1000; this.getTouchJournal = opts?.getTouchJournal; } start() { if (!this.enabled) return; if (this.timer) return; this.timer = setInterval(() => { this.flushTick(); }, this.intervalMs); this.snapshotTimer = setInterval(() => { this.emitSnapshots(); }, this.snapshotIntervalMs); this.retentionTimer = setInterval(() => { try { this.db.trimWalByAge(this.metricsStream, this.retentionMs); } catch {} }, 60000); const lagIntervalMs = 100; this.lagExpectedMs = Date.now() + lagIntervalMs; this.lagMaxMs = 0; this.lagSumMs = 0; this.lagSamples = 0; this.lagTimer = setInterval(() => { const now = Date.now(); const lag = Math.max(0, now - this.lagExpectedMs); this.lagMaxMs = Math.max(this.lagMaxMs, lag); this.lagSumMs += lag; this.lagSamples += 1; this.lagExpectedMs += lagIntervalMs; if (this.lagExpectedMs < now - 5 * lagIntervalMs) this.lagExpectedMs = now + lagIntervalMs; }, lagIntervalMs); } stop() { if (this.timer) clearInterval(this.timer); if (this.snapshotTimer) clearInterval(this.snapshotTimer); if (this.retentionTimer) clearInterval(this.retentionTimer); if (this.lagTimer) clearInterval(this.lagTimer); this.timer = null; this.snapshotTimer = null; this.retentionTimer = null; this.lagTimer = null; } ensureStreamResult() { if (!this.enabled) return Result14.ok(undefined); const existing = this.db.getStream(this.metricsStream); if (existing) { if (String(existing.content_type) !== "application/json") { return Result14.err({ kind: "live_metrics_stream_content_type_mismatch", message: `live metrics stream content-type mismatch: ${existing.content_type}` }); } if ((existing.stream_flags & STREAM_FLAG_TOUCH) === 0) this.db.addStreamFlags(this.metricsStream, STREAM_FLAG_TOUCH); return Result14.ok(undefined); } this.db.ensureStream(this.metricsStream, { contentType: "application/json", streamFlags: STREAM_FLAG_TOUCH }); return Result14.ok(undefined); } get(stream, touchCfg) { const existing = this.counters.get(stream); if (existing) return existing; const c = defaultCounters(touchCfg); this.counters.set(stream, c); return c; } ensure(stream) { const existing = this.counters.get(stream); if (existing) return existing; const c = defaultCounters({ enabled: true }); this.counters.set(stream, c); return c; } recordInterpreterError(stream, touchCfg) { const c = this.get(stream, touchCfg); c.interpreter.errors += 1; } recordInterpreterBatch(args) { const c = this.get(args.stream, args.touchCfg); c.touch.coarseIntervalMs = args.touchCfg.coarseIntervalMs ?? c.touch.coarseIntervalMs; c.touch.coalesceWindowMs = args.touchCfg.touchCoalesceWindowMs ?? c.touch.coalesceWindowMs; c.touch.mode = args.touchMode; c.touch.hotFineKeys = Math.max(c.touch.hotFineKeys, Math.max(0, Math.floor(args.hotFineKeys ?? 0))); c.touch.hotTemplates = Math.max(c.touch.hotTemplates, Math.max(0, Math.floor(args.hotTemplates ?? 0))); c.touch.hotFineKeysActive = Math.max(c.touch.hotFineKeysActive, Math.max(0, Math.floor(args.hotFineKeysActive ?? 0))); c.touch.hotFineKeysGrace = Math.max(c.touch.hotFineKeysGrace, Math.max(0, Math.floor(args.hotFineKeysGrace ?? 0))); c.touch.hotTemplatesActive = Math.max(c.touch.hotTemplatesActive, Math.max(0, Math.floor(args.hotTemplatesActive ?? 0))); c.touch.hotTemplatesGrace = Math.max(c.touch.hotTemplatesGrace, Math.max(0, Math.floor(args.hotTemplatesGrace ?? 0))); c.touch.fineWaitersActive = Math.max(c.touch.fineWaitersActive, Math.max(0, Math.floor(args.fineWaitersActive ?? 0))); c.touch.coarseWaitersActive = Math.max(c.touch.coarseWaitersActive, Math.max(0, Math.floor(args.coarseWaitersActive ?? 0))); c.touch.broadFineWaitersActive = Math.max(c.touch.broadFineWaitersActive, Math.max(0, Math.floor(args.broadFineWaitersActive ?? 0))); c.interpreter.eventsIn += Math.max(0, args.rowsRead); c.interpreter.changesOut += Math.max(0, args.changes); c.interpreter.lagSourceOffsets = Math.max(c.interpreter.lagSourceOffsets, Math.max(0, args.lagSourceOffsets)); c.interpreter.scannedBatches += 1; if (args.scannedButEmitted0) c.interpreter.scannedButEmitted0Batches += 1; if (args.noInterestFastForward) c.interpreter.noInterestFastForwardBatches += 1; c.interpreter.interpretedThroughDelta += Math.max(0, Math.floor(args.interpretedThroughDelta ?? 0)); c.interpreter.touchesEmittedDelta += Math.max(0, Math.floor(args.touchesEmittedDelta ?? 0)); if (args.commitLagMs != null && Number.isFinite(args.commitLagMs) && args.commitLagMs >= 0) { c.interpreter.commitLagSamples += 1; c.interpreter.commitLagMsSum += args.commitLagMs; c.interpreter.commitLagHist.record(args.commitLagMs); } c.touch.fineTouchesDroppedDueToBudget += Math.max(0, args.fineTouchesDroppedDueToBudget ?? 0); c.touch.fineTouchesSkippedColdTemplate += Math.max(0, args.fineTouchesSkippedColdTemplate ?? 0); c.touch.fineTouchesSkippedColdKey += Math.max(0, args.fineTouchesSkippedColdKey ?? 0); c.touch.fineTouchesSkippedTemplateBucket += Math.max(0, args.fineTouchesSkippedTemplateBucket ?? 0); if (args.fineTouchesSuppressedDueToLag) c.touch.fineTouchesSuppressedBatchesDueToLag += 1; c.touch.fineTouchesSuppressedMsDueToLag += Math.max(0, args.fineTouchesSuppressedDueToLagMs ?? 0); if (args.fineTouchesSuppressedDueToBudget) c.touch.fineTouchesSuppressedBatchesDueToBudget += 1; const unique = new Set; let table = 0; let tpl = 0; for (const t of args.touches) { unique.add(t.keyId >>> 0); if (t.kind === "table") table++; else tpl++; } c.touch.touchesEmitted += args.touches.length; c.touch.uniqueKeysTouched += unique.size; c.touch.tableTouchesEmitted += table; c.touch.templateTouchesEmitted += tpl; } recordWait(stream, touchCfg, keysCount, outcome, latencyMs) { const c = this.get(stream, touchCfg); c.wait.calls += 1; c.wait.keysWatchedTotal += Math.max(0, keysCount); c.wait.latencySumMs += Math.max(0, latencyMs); c.wait.latencyHist.record(latencyMs); if (outcome === "touched") c.wait.touched += 1; else if (outcome === "timeout") c.wait.timeout += 1; else c.wait.stale += 1; if (outcome === "stale") c.touch.staleResponses += 1; } recordBaseWalGc(stream, args) { const c = this.ensure(stream); c.gc.baseWalGcCalls += 1; c.gc.baseWalGcDeletedRows += Math.max(0, args.deletedRows); c.gc.baseWalGcDeletedBytes += Math.max(0, args.deletedBytes); c.gc.baseWalGcMsSum += Math.max(0, args.durationMs); c.gc.baseWalGcMsMax = Math.max(c.gc.baseWalGcMsMax, Math.max(0, args.durationMs)); } async emitLifecycle(events) { if (!this.enabled) return; if (events.length === 0) return; const rows = events.map((e) => ({ routingKey: new TextEncoder().encode(`${e.stream}|${e.type}`), contentType: "application/json", payload: new TextEncoder().encode(JSON.stringify({ ...e, liveSystemVersion: "v2", instanceId: this.instanceId, region: this.region })) })); for (const e of events) { const c = this.ensure(e.stream); if (e.type === "live.template_activated") c.templates.activated += 1; else if (e.type === "live.template_retired") c.templates.retired += 1; else if (e.type === "live.template_evicted") c.templates.evicted += 1; } try { await this.ingest.appendInternal({ stream: this.metricsStream, baseAppendMs: BigInt(Date.now()), rows, contentType: "application/json" }); } catch {} } recordActivationDenied(stream, touchCfg, n = 1) { const c = this.get(stream, touchCfg); c.templates.activationDenied += Math.max(0, n); } async flushTick() { if (!this.enabled) return; const nowMs = Date.now(); const clampBigInt = (v) => { if (v <= 0n) return 0; const max = BigInt(Number.MAX_SAFE_INTEGER); return v > max ? Number.MAX_SAFE_INTEGER : Number(v); }; const states = this.db.listStreamInterpreters(); if (states.length === 0) return; const rows = []; const encoder = new TextEncoder; const loopLagMax = this.lagMaxMs; const loopLagAvg = this.lagSamples > 0 ? this.lagSumMs / this.lagSamples : 0; this.lagMaxMs = 0; this.lagSumMs = 0; this.lagSamples = 0; for (const st of states) { const stream = st.stream; const regRow = this.db.getStream(stream); if (!regRow) continue; const touchCfg = (() => { try { const row = this.db.getSchemaRegistry(stream); if (!row) return null; const raw = JSON.parse(row.registry_json); const cfg = raw?.interpreter?.touch; if (!cfg || !cfg.enabled) return null; return cfg; } catch { return null; } })(); if (!touchCfg) continue; const c = this.get(stream, touchCfg); const journal = this.getTouchJournal?.(stream) ?? null; const waitActive = journal?.meta.activeWaiters ?? 0; const tailSeq = regRow.next_offset > 0n ? regRow.next_offset - 1n : -1n; const interpretedThrough = st.interpreted_through; const gcThrough = interpretedThrough < regRow.uploaded_through ? interpretedThrough : regRow.uploaded_through; const backlog = tailSeq >= interpretedThrough ? tailSeq - interpretedThrough : 0n; const backlogNum = backlog > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(backlog); let walOldestOffset = null; try { const oldest = this.db.getWalOldestOffset(stream); walOldestOffset = oldest == null ? null : encodeOffset(regRow.epoch, oldest); } catch { walOldestOffset = null; } const activeTemplates = (() => { try { const row = this.db.db.query(`SELECT COUNT(*) as cnt FROM live_templates WHERE stream=? AND state='active';`).get(stream); return Number(row?.cnt ?? 0); } catch { return 0; } })(); const tick = { type: "live.tick", ts: nowIso2(nowMs), stream, liveSystemVersion: "v2", instanceId: this.instanceId, region: this.region, touch: { coarseIntervalMs: c.touch.coarseIntervalMs, coalesceWindowMs: c.touch.coalesceWindowMs, mode: c.touch.mode, hotFineKeys: c.touch.hotFineKeys, hotTemplates: c.touch.hotTemplates, hotFineKeysActive: c.touch.hotFineKeysActive, hotFineKeysGrace: c.touch.hotFineKeysGrace, hotTemplatesActive: c.touch.hotTemplatesActive, hotTemplatesGrace: c.touch.hotTemplatesGrace, fineWaitersActive: c.touch.fineWaitersActive, coarseWaitersActive: c.touch.coarseWaitersActive, broadFineWaitersActive: c.touch.broadFineWaitersActive, touchesEmitted: c.touch.touchesEmitted, uniqueKeysTouched: c.touch.uniqueKeysTouched, tableTouchesEmitted: c.touch.tableTouchesEmitted, templateTouchesEmitted: c.touch.templateTouchesEmitted, staleResponses: c.touch.staleResponses, fineTouchesDroppedDueToBudget: c.touch.fineTouchesDroppedDueToBudget, fineTouchesSkippedColdTemplate: c.touch.fineTouchesSkippedColdTemplate, fineTouchesSkippedColdKey: c.touch.fineTouchesSkippedColdKey, fineTouchesSkippedTemplateBucket: c.touch.fineTouchesSkippedTemplateBucket, fineTouchesSuppressedBatchesDueToLag: c.touch.fineTouchesSuppressedBatchesDueToLag, fineTouchesSuppressedSecondsDueToLag: c.touch.fineTouchesSuppressedMsDueToLag / 1000, fineTouchesSuppressedBatchesDueToBudget: c.touch.fineTouchesSuppressedBatchesDueToBudget, cursor: journal?.meta.cursor ?? null, epoch: journal?.meta.epoch ?? null, generation: journal?.meta.generation ?? null, pendingKeys: journal?.meta.pendingKeys ?? 0, overflowBuckets: journal?.meta.overflowBuckets ?? 0 }, templates: { active: activeTemplates, activated: c.templates.activated, retired: c.templates.retired, evicted: c.templates.evicted, activationDenied: c.templates.activationDenied }, wait: { calls: c.wait.calls, keysWatchedTotal: c.wait.keysWatchedTotal, avgKeysPerCall: c.wait.calls > 0 ? c.wait.keysWatchedTotal / c.wait.calls : 0, touched: c.wait.touched, timeout: c.wait.timeout, stale: c.wait.stale, avgLatencyMs: c.wait.calls > 0 ? c.wait.latencySumMs / c.wait.calls : 0, p95LatencyMs: c.wait.latencyHist.p95(), activeWaiters: waitActive, timeoutsFired: journal?.interval.timeoutsFired ?? 0, timeoutSweeps: journal?.interval.timeoutSweeps ?? 0, timeoutSweepMsSum: journal?.interval.timeoutSweepMsSum ?? 0, timeoutSweepMsMax: journal?.interval.timeoutSweepMsMax ?? 0, notifyWakeups: journal?.interval.notifyWakeups ?? 0, notifyFlushes: journal?.interval.notifyFlushes ?? 0, notifyWakeMsSum: journal?.interval.notifyWakeMsSum ?? 0, notifyWakeMsMax: journal?.interval.notifyWakeMsMax ?? 0, timeoutHeapSize: journal?.interval.heapSize ?? 0 }, interpreter: { eventsIn: c.interpreter.eventsIn, changesOut: c.interpreter.changesOut, errors: c.interpreter.errors, lagSourceOffsets: c.interpreter.lagSourceOffsets, scannedBatches: c.interpreter.scannedBatches, scannedButEmitted0Batches: c.interpreter.scannedButEmitted0Batches, noInterestFastForwardBatches: c.interpreter.noInterestFastForwardBatches, interpretedThroughDelta: c.interpreter.interpretedThroughDelta, touchesEmittedDelta: c.interpreter.touchesEmittedDelta, commitLagMsAvg: c.interpreter.commitLagSamples > 0 ? c.interpreter.commitLagMsSum / c.interpreter.commitLagSamples : 0, commitLagMsP50: c.interpreter.commitLagHist.p50(), commitLagMsP95: c.interpreter.commitLagHist.p95(), commitLagMsP99: c.interpreter.commitLagHist.p99() }, base: { tailOffset: encodeOffset(regRow.epoch, tailSeq), nextOffset: encodeOffset(regRow.epoch, regRow.next_offset), sealedThrough: encodeOffset(regRow.epoch, regRow.sealed_through), uploadedThrough: encodeOffset(regRow.epoch, regRow.uploaded_through), interpretedThrough: encodeOffset(regRow.epoch, interpretedThrough), gcThrough: encodeOffset(regRow.epoch, gcThrough), walOldestOffset, walRetainedRows: clampBigInt(regRow.wal_rows), walRetainedBytes: clampBigInt(regRow.wal_bytes), gc: { calls: c.gc.baseWalGcCalls, deletedRows: c.gc.baseWalGcDeletedRows, deletedBytes: c.gc.baseWalGcDeletedBytes, msSum: c.gc.baseWalGcMsSum, msMax: c.gc.baseWalGcMsMax }, backlogSourceOffsets: backlogNum }, process: { eventLoopLagMsMax: loopLagMax, eventLoopLagMsAvg: loopLagAvg } }; rows.push({ routingKey: encoder.encode(`${stream}|live.tick`), contentType: "application/json", payload: encoder.encode(JSON.stringify(tick)) }); c.touch.hotFineKeys = 0; c.touch.hotTemplates = 0; c.touch.hotFineKeysActive = 0; c.touch.hotFineKeysGrace = 0; c.touch.hotTemplatesActive = 0; c.touch.hotTemplatesGrace = 0; c.touch.fineWaitersActive = 0; c.touch.coarseWaitersActive = 0; c.touch.broadFineWaitersActive = 0; c.touch.touchesEmitted = 0; c.touch.uniqueKeysTouched = 0; c.touch.tableTouchesEmitted = 0; c.touch.templateTouchesEmitted = 0; c.touch.staleResponses = 0; c.touch.fineTouchesDroppedDueToBudget = 0; c.touch.fineTouchesSkippedColdTemplate = 0; c.touch.fineTouchesSkippedColdKey = 0; c.touch.fineTouchesSkippedTemplateBucket = 0; c.touch.fineTouchesSuppressedBatchesDueToLag = 0; c.touch.fineTouchesSuppressedMsDueToLag = 0; c.touch.fineTouchesSuppressedBatchesDueToBudget = 0; c.touch.mode = "idle"; c.templates.activated = 0; c.templates.retired = 0; c.templates.evicted = 0; c.templates.activationDenied = 0; c.wait.calls = 0; c.wait.keysWatchedTotal = 0; c.wait.touched = 0; c.wait.timeout = 0; c.wait.stale = 0; c.wait.latencySumMs = 0; c.wait.latencyHist.reset(); c.interpreter.eventsIn = 0; c.interpreter.changesOut = 0; c.interpreter.errors = 0; c.interpreter.lagSourceOffsets = 0; c.interpreter.scannedBatches = 0; c.interpreter.scannedButEmitted0Batches = 0; c.interpreter.noInterestFastForwardBatches = 0; c.interpreter.interpretedThroughDelta = 0; c.interpreter.touchesEmittedDelta = 0; c.interpreter.commitLagSamples = 0; c.interpreter.commitLagMsSum = 0; c.interpreter.commitLagHist.reset(); c.gc.baseWalGcCalls = 0; c.gc.baseWalGcDeletedRows = 0; c.gc.baseWalGcDeletedBytes = 0; c.gc.baseWalGcMsSum = 0; c.gc.baseWalGcMsMax = 0; } if (rows.length === 0) return; try { await this.ingest.appendInternal({ stream: this.metricsStream, baseAppendMs: BigInt(nowMs), rows, contentType: "application/json" }); } catch {} } async emitSnapshots() { if (!this.enabled) return; const nowMs = Date.now(); const streams = this.db.listStreamInterpreters().map((r) => r.stream); if (streams.length === 0) return; const encoder = new TextEncoder; const rows = []; for (const stream of streams) { let templates = []; try { templates = this.db.db.query(`SELECT template_id, entity, fields_json, last_seen_at_ms, state FROM live_templates WHERE stream=? AND state='active' ORDER BY entity ASC, template_id ASC;`).all(stream); } catch { continue; } const snapshotId = `s-${stream}-${nowMs}`; const activeTemplates = templates.length; rows.push({ routingKey: encoder.encode(`${stream}|live.templates_snapshot_start`), contentType: "application/json", payload: encoder.encode(JSON.stringify({ type: "live.templates_snapshot_start", ts: nowIso2(nowMs), stream, liveSystemVersion: "v2", instanceId: this.instanceId, region: this.region, snapshotId, activeTemplates, chunkSize: this.snapshotChunkSize })) }); let chunkIndex = 0; for (let i = 0;i < templates.length; i += this.snapshotChunkSize) { const slice = templates.slice(i, i + this.snapshotChunkSize); const payloadTemplates = slice.map((t) => { const templateId = String(t.template_id); const entity = String(t.entity); let fields = []; try { const f = JSON.parse(String(t.fields_json)); if (Array.isArray(f)) fields = f.map(String); } catch {} const lastSeenAgoMs = Math.max(0, nowMs - Number(t.last_seen_at_ms)); return { templateId, entity, fields, lastSeenAgoMs, state: "active" }; }); rows.push({ routingKey: encoder.encode(`${stream}|live.templates_snapshot_chunk`), contentType: "application/json", payload: encoder.encode(JSON.stringify({ type: "live.templates_snapshot_chunk", ts: nowIso2(nowMs), stream, liveSystemVersion: "v2", instanceId: this.instanceId, region: this.region, snapshotId, chunkIndex, templates: payloadTemplates })) }); chunkIndex++; } rows.push({ routingKey: encoder.encode(`${stream}|live.templates_snapshot_end`), contentType: "application/json", payload: encoder.encode(JSON.stringify({ type: "live.templates_snapshot_end", ts: nowIso2(nowMs), stream, liveSystemVersion: "v2", instanceId: this.instanceId, region: this.region, snapshotId })) }); } if (rows.length === 0) return; try { await this.ingest.appendInternal({ stream: this.metricsStream, baseAppendMs: BigInt(nowMs), rows, contentType: "application/json" }); } catch {} } } // src/touch/touch_journal.ts function u32(x) { return x >>> 0; } function mix32(x) { let y = u32(x); y ^= y >>> 16; y = Math.imul(y, 2246822507) >>> 0; y ^= y >>> 13; y = Math.imul(y, 3266489909) >>> 0; y ^= y >>> 16; return y >>> 0; } function newEpochHex16() { const buf = new Uint32Array(2); crypto.getRandomValues(buf); return buf[0].toString(16).padStart(8, "0") + buf[1].toString(16).padStart(8, "0"); } function parseTouchCursor(raw) { const s = raw.trim(); if (s === "") return null; const idx = s.indexOf(":"); if (idx <= 0) return null; const epoch = s.slice(0, idx); const genRaw = s.slice(idx + 1); if (!/^[0-9a-f]{16}$/i.test(epoch)) return null; if (!/^[0-9]+$/.test(genRaw)) return null; const gen = Number(genRaw); if (!Number.isFinite(gen) || gen < 0) return null; return { epoch: epoch.toLowerCase(), generation: Math.floor(gen) }; } function formatTouchCursor(epoch, generation) { return `${epoch}:${Math.max(0, Math.floor(generation))}`; } class TouchJournal { epoch; generation; bucketMs; coalesceMs; k; mask; lastSet; pending = new Set; pendingBucketStartMs = 0; pendingMaxSourceOffsetSeq = -1n; lastFlushedSourceOffsetSeq = -1n; overflow = false; overflowBuckets = 0; lastOverflowGeneration = 0; lastFlushAtMs = 0; lastBucketStartMs = 0; flushIntervalsLast10s = []; flushTimer = null; byKey = new Map; broad = new Set; activeWaiters = 0; deadlineHeap = []; timeoutTimer = null; scheduledDeadlineMs = null; interval = { timeoutsFired: 0, timeoutSweeps: 0, timeoutSweepMsSum: 0, timeoutSweepMsMax: 0, notifyWakeups: 0, notifyFlushes: 0, notifyWakeMsSum: 0, notifyWakeMsMax: 0, heapSize: 0 }; totals = { timeoutsFired: 0, timeoutSweeps: 0, timeoutSweepMsSum: 0, timeoutSweepMsMax: 0, notifyWakeups: 0, notifyFlushes: 0, notifyWakeMsSum: 0, notifyWakeMsMax: 0, flushes: 0 }; pendingMaxKeys; keyIndexMaxKeys; constructor(opts) { this.epoch = newEpochHex16(); this.generation = 0; this.bucketMs = Math.max(1, Math.floor(opts.bucketMs)); this.coalesceMs = this.bucketMs; const pow2 = Math.max(10, Math.min(30, Math.floor(opts.filterPow2))); const size = 1 << pow2; this.k = Math.max(1, Math.min(8, Math.floor(opts.k))); this.mask = size - 1; this.lastSet = new Uint32Array(size); this.pendingMaxKeys = Math.max(1, Math.floor(opts.pendingMaxKeys)); this.keyIndexMaxKeys = Math.max(1, Math.floor(opts.keyIndexMaxKeys)); } stop() { if (this.flushTimer) clearTimeout(this.flushTimer); if (this.timeoutTimer) clearTimeout(this.timeoutTimer); this.flushTimer = null; this.timeoutTimer = null; this.scheduledDeadlineMs = null; this.pending.clear(); this.pendingBucketStartMs = 0; this.pendingMaxSourceOffsetSeq = -1n; this.lastFlushedSourceOffsetSeq = -1n; this.lastFlushAtMs = 0; this.lastBucketStartMs = 0; this.flushIntervalsLast10s.length = 0; this.byKey.clear(); this.broad.clear(); this.deadlineHeap.length = 0; this.activeWaiters = 0; } getEpoch() { return this.epoch; } getGeneration() { return this.generation >>> 0; } getCursor() { return formatTouchCursor(this.epoch, this.getGeneration()); } getLastFlushedSourceOffsetSeq() { return this.lastFlushedSourceOffsetSeq; } getActiveWaiters() { return this.activeWaiters; } snapshotAndResetIntervalStats() { const out = { ...this.interval, heapSize: this.deadlineHeap.length }; this.interval = { timeoutsFired: 0, timeoutSweeps: 0, timeoutSweepMsSum: 0, timeoutSweepMsMax: 0, notifyWakeups: 0, notifyFlushes: 0, notifyWakeMsSum: 0, notifyWakeMsMax: 0, heapSize: 0 }; return out; } getTotalStats() { return { ...this.totals }; } getMeta() { const nowMs = Date.now(); this.pruneFlushIntervals(nowMs); const intervals = this.flushIntervalsLast10s.map((x) => x.intervalMs); return { mode: "memory", cursor: this.getCursor(), epoch: this.epoch, generation: this.getGeneration(), bucketMs: this.bucketMs, coalesceMs: this.coalesceMs, filterSize: this.lastSet.length, k: this.k, pendingKeys: this.pending.size, overflowBuckets: this.overflowBuckets, activeWaiters: this.activeWaiters, bucketMaxSourceOffsetSeq: this.lastFlushedSourceOffsetSeq.toString(), lastFlushAtMs: this.lastFlushAtMs, flushIntervalMsMaxLast10s: intervals.length > 0 ? Math.max(...intervals) : 0, flushIntervalMsP95Last10s: percentile(intervals, 0.95) }; } touch(keyId, sourceOffsetSeq) { if (this.pending.size === 0 && !this.overflow && this.pendingBucketStartMs <= 0) { this.pendingBucketStartMs = Date.now(); } if (this.pending.size >= this.pendingMaxKeys) { this.overflow = true; } else { this.pending.add(u32(keyId)); } if (typeof sourceOffsetSeq === "bigint" && sourceOffsetSeq > this.pendingMaxSourceOffsetSeq) { this.pendingMaxSourceOffsetSeq = sourceOffsetSeq; } this.ensureFlushScheduled(); } setCoalesceMs(ms) { const next = Math.max(1, Math.min(this.bucketMs, Math.floor(ms))); this.coalesceMs = next; } maybeTouchedSince(keyId, sinceGeneration) { const since = u32(sinceGeneration); if (since < u32(this.lastOverflowGeneration)) return true; const h1 = u32(keyId); let h2 = mix32(h1); if (h2 === 0) h2 = 2654435769; let min = 4294967295; for (let i = 0;i < this.k; i++) { const pos = u32(h1 + Math.imul(i, h2)) & this.mask; const g = this.lastSet[pos]; if (g < min) min = g; } return u32(min) > since; } maybeTouchedSinceAny(keyIds, sinceGeneration) { const since = u32(sinceGeneration); if (since < u32(this.lastOverflowGeneration)) return true; for (let i = 0;i < keyIds.length; i++) { if (this.maybeTouchedSince(keyIds[i], since)) return true; } return false; } waitForAny(args) { if (args.keys.length === 0) return Promise.resolve(null); if (args.signal?.aborted) return Promise.resolve(null); const timeoutMs = Math.max(0, Math.floor(args.timeoutMs)); if (timeoutMs <= 0) return Promise.resolve(null); const keys = Array.from(new Set(args.keys.map(u32))); const broad = keys.length > this.keyIndexMaxKeys; return new Promise((resolve2) => { const waiter = { afterGeneration: u32(args.afterGeneration), keys, broad, deadlineMs: Date.now() + timeoutMs, heapIndex: -1, done: false, cleanup: (hit) => { if (waiter.done) return; waiter.done = true; if (waiter.broad) { this.broad.delete(waiter); } else { for (const k of waiter.keys) { const s = this.byKey.get(k); if (!s) continue; s.delete(waiter); if (s.size === 0) this.byKey.delete(k); } } this.activeWaiters = Math.max(0, this.activeWaiters - 1); const removedRoot = this.heapRemove(waiter); if (args.signal) args.signal.removeEventListener("abort", onAbort); if (removedRoot) this.rescheduleTimeoutTimer(); resolve2(hit); } }; if (waiter.broad) { this.broad.add(waiter); } else { for (const k of waiter.keys) { const set = this.byKey.get(k) ?? new Set; set.add(waiter); this.byKey.set(k, set); } } this.activeWaiters += 1; const onAbort = () => waiter.cleanup(null); if (args.signal) args.signal.addEventListener("abort", onAbort, { once: true }); this.heapPush(waiter); this.rescheduleTimeoutTimer(); }); } ensureFlushScheduled() { if (this.flushTimer) return; this.flushTimer = setTimeout(() => this.flushBucket(), this.coalesceMs); } flushBucket() { this.flushTimer = null; const hasTouches = this.pending.size > 0 || this.overflow; if (!hasTouches) return; this.generation = u32(this.generation + 1); const gen = this.getGeneration(); const bucketMaxSourceOffsetSeq = this.pendingMaxSourceOffsetSeq; if (bucketMaxSourceOffsetSeq > this.lastFlushedSourceOffsetSeq) this.lastFlushedSourceOffsetSeq = bucketMaxSourceOffsetSeq; const flushAtMs = Date.now(); const bucketStartMs = this.pendingBucketStartMs > 0 ? this.pendingBucketStartMs : flushAtMs; if (this.lastFlushAtMs > 0 && flushAtMs >= this.lastFlushAtMs) { this.flushIntervalsLast10s.push({ atMs: flushAtMs, intervalMs: flushAtMs - this.lastFlushAtMs }); this.pruneFlushIntervals(flushAtMs); } this.lastFlushAtMs = flushAtMs; this.lastBucketStartMs = bucketStartMs; this.totals.flushes += 1; if (this.overflow) { this.overflowBuckets += 1; this.lastOverflowGeneration = gen; } for (const keyId of this.pending) { const h1 = u32(keyId); let h2 = mix32(h1); if (h2 === 0) h2 = 2654435769; for (let i = 0;i < this.k; i++) { const pos = u32(h1 + Math.imul(i, h2)) & this.mask; this.lastSet[pos] = gen; } } if (this.overflow) { const wakeStartMs = Date.now(); let wakeups = 0; const all = []; for (const s of this.byKey.values()) for (const w of s) all.push(w); for (const w of this.broad) all.push(w); for (const w of all) { if (w.done) continue; if (gen > w.afterGeneration) { wakeups += 1; w.cleanup({ generation: gen, keyId: 0, bucketMaxSourceOffsetSeq, flushAtMs, bucketStartMs }); } } if (wakeups > 0) { const wakeMs = Date.now() - wakeStartMs; this.interval.notifyWakeups += wakeups; this.interval.notifyFlushes += 1; this.interval.notifyWakeMsSum += wakeMs; this.interval.notifyWakeMsMax = Math.max(this.interval.notifyWakeMsMax, wakeMs); this.totals.notifyWakeups += wakeups; this.totals.notifyFlushes += 1; this.totals.notifyWakeMsSum += wakeMs; this.totals.notifyWakeMsMax = Math.max(this.totals.notifyWakeMsMax, wakeMs); } } else { const wakeStartMs = Date.now(); let wakeups = 0; for (const keyId of this.pending) { const set = this.byKey.get(keyId); if (!set || set.size === 0) continue; for (const w of set) { if (w.done) continue; if (gen > w.afterGeneration) { wakeups += 1; w.cleanup({ generation: gen, keyId, bucketMaxSourceOffsetSeq, flushAtMs, bucketStartMs }); } } } if (this.broad.size > 0) { for (const w of this.broad) { if (w.done) continue; if (gen <= w.afterGeneration) continue; let hit = false; for (let i = 0;i < w.keys.length; i++) { if (this.maybeTouchedSince(w.keys[i], w.afterGeneration)) { hit = true; break; } } if (hit) { wakeups += 1; w.cleanup({ generation: gen, keyId: 0, bucketMaxSourceOffsetSeq, flushAtMs, bucketStartMs }); } } } if (wakeups > 0) { const wakeMs = Date.now() - wakeStartMs; this.interval.notifyWakeups += wakeups; this.interval.notifyFlushes += 1; this.interval.notifyWakeMsSum += wakeMs; this.interval.notifyWakeMsMax = Math.max(this.interval.notifyWakeMsMax, wakeMs); this.totals.notifyWakeups += wakeups; this.totals.notifyFlushes += 1; this.totals.notifyWakeMsSum += wakeMs; this.totals.notifyWakeMsMax = Math.max(this.totals.notifyWakeMsMax, wakeMs); } } this.pending.clear(); this.pendingBucketStartMs = 0; this.pendingMaxSourceOffsetSeq = -1n; this.overflow = false; } getLastFlushAtMs() { return this.lastFlushAtMs; } getLastBucketStartMs() { return this.lastBucketStartMs; } pruneFlushIntervals(nowMs) { const cutoff = nowMs - 1e4; while (this.flushIntervalsLast10s.length > 0 && this.flushIntervalsLast10s[0].atMs < cutoff) { this.flushIntervalsLast10s.shift(); } } rescheduleTimeoutTimer() { const next = this.deadlineHeap[0]; if (!next) { if (this.timeoutTimer) clearTimeout(this.timeoutTimer); this.timeoutTimer = null; this.scheduledDeadlineMs = null; return; } if (this.timeoutTimer && this.scheduledDeadlineMs != null && this.scheduledDeadlineMs === next.deadlineMs) return; if (this.timeoutTimer) clearTimeout(this.timeoutTimer); this.scheduledDeadlineMs = next.deadlineMs; const delayMs = Math.max(0, next.deadlineMs - Date.now()); this.timeoutTimer = setTimeout(() => this.expireDueWaiters(), delayMs); } expireDueWaiters() { this.timeoutTimer = null; this.scheduledDeadlineMs = null; const start = Date.now(); const now = start; let expired = 0; for (;; ) { const head = this.deadlineHeap[0]; if (!head) break; if (head.deadlineMs > now) break; const w = this.heapPopMin(); if (!w) break; if (w.done) continue; expired += 1; w.cleanup(null); } if (expired > 0) { const sweepMs = Date.now() - start; this.interval.timeoutsFired += expired; this.interval.timeoutSweeps += 1; this.interval.timeoutSweepMsSum += sweepMs; this.interval.timeoutSweepMsMax = Math.max(this.interval.timeoutSweepMsMax, sweepMs); this.totals.timeoutsFired += expired; this.totals.timeoutSweeps += 1; this.totals.timeoutSweepMsSum += sweepMs; this.totals.timeoutSweepMsMax = Math.max(this.totals.timeoutSweepMsMax, sweepMs); } this.rescheduleTimeoutTimer(); } heapSwap(i, j) { const a = this.deadlineHeap[i]; const b = this.deadlineHeap[j]; this.deadlineHeap[i] = b; this.deadlineHeap[j] = a; a.heapIndex = j; b.heapIndex = i; } heapLess(i, j) { const a = this.deadlineHeap[i]; const b = this.deadlineHeap[j]; return a.deadlineMs < b.deadlineMs; } heapSiftUp(i) { let idx = i; while (idx > 0) { const parent = idx - 1 >> 1; if (!this.heapLess(idx, parent)) break; this.heapSwap(idx, parent); idx = parent; } } heapSiftDown(i) { let idx = i; for (;; ) { const left = idx * 2 + 1; const right = left + 1; if (left >= this.deadlineHeap.length) break; let smallest = left; if (right < this.deadlineHeap.length && this.heapLess(right, left)) smallest = right; if (!this.heapLess(smallest, idx)) break; this.heapSwap(idx, smallest); idx = smallest; } } heapPush(w) { if (w.heapIndex >= 0) return; w.heapIndex = this.deadlineHeap.length; this.deadlineHeap.push(w); this.heapSiftUp(w.heapIndex); } heapRemove(w) { const idx = w.heapIndex; if (idx < 0) return false; const lastIdx = this.deadlineHeap.length - 1; const removedRoot = idx === 0; if (idx !== lastIdx) this.heapSwap(idx, lastIdx); this.deadlineHeap.pop(); w.heapIndex = -1; if (idx < this.deadlineHeap.length) { this.heapSiftDown(idx); this.heapSiftUp(idx); } return removedRoot; } heapPopMin() { if (this.deadlineHeap.length === 0) return null; const w = this.deadlineHeap[0]; const last = this.deadlineHeap.length - 1; if (last === 0) { this.deadlineHeap.pop(); w.heapIndex = -1; return w; } this.heapSwap(0, last); this.deadlineHeap.pop(); w.heapIndex = -1; this.heapSiftDown(0); return w; } } function percentile(values, p) { if (values.length === 0) return 0; const sorted = [...values].sort((a, b) => a - b); const idx = Math.max(0, Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * p))); return sorted[idx] ?? 0; } // src/touch/manager.ts import { Result as Result15 } from "better-result"; var BASE_WAL_GC_INTERVAL_MS = (() => { const raw = process.env.DS_BASE_WAL_GC_INTERVAL_MS; if (raw == null || raw.trim() === "") return 1000; const n = Number(raw); if (!Number.isFinite(n) || n < 0) { console.error(`invalid DS_BASE_WAL_GC_INTERVAL_MS: ${raw}`); return 1000; } return Math.floor(n); })(); var BASE_WAL_GC_CHUNK_OFFSETS2 = (() => { const raw = process.env.DS_BASE_WAL_GC_CHUNK_OFFSETS; if (raw == null || raw.trim() === "") return 1e5; const n = Number(raw); if (!Number.isFinite(n) || n <= 0) { console.error(`invalid DS_BASE_WAL_GC_CHUNK_OFFSETS: ${raw}`); return 1e5; } return Math.floor(n); })(); var HOT_INTEREST_MAX_KEYS = 64; class TouchInterpreterManager { cfg; db; registry; pool; timer = null; running = false; stopping = false; dirty = new Set; failures = new FailureTracker(1024); lastBaseWalGc = new LruCache(1024); templates; liveMetrics; lastTemplateGcMsByStream = new LruCache(1024); journals = new Map; fineLagCoarseOnlyByStream = new Map; touchModeByStream = new Map; fineTokenBucketsByStream = new Map; hotFineByStream = new Map; lagSourceOffsetsByStream = new Map; restrictedTemplateBucketStateByStream = new Map; runtimeTotalsByStream = new Map; zeroRowBacklogStreakByStream = new Map; streamScanCursor = 0; restartWorkerPoolRequested = false; lastWorkerPoolRestartAtMs = 0; constructor(cfg, db, ingest, notifier, registry, backpressure) { this.cfg = cfg; this.db = db; this.registry = registry; this.pool = new TouchInterpreterWorkerPool(cfg, cfg.interpreterWorkers); this.templates = new LiveTemplateRegistry(db); this.liveMetrics = new LiveMetricsV2(db, ingest, { getTouchJournal: (stream) => { const j = this.journals.get(stream); if (!j) return null; return { meta: j.getMeta(), interval: j.snapshotAndResetIntervalStats() }; } }); } start() { if (this.timer) return; this.stopping = false; this.pool.start(); this.seedInterpretersFromRegistry(); const liveMetricsRes = this.liveMetrics.ensureStreamResult(); if (Result15.isError(liveMetricsRes)) { console.error("touch live metrics stream validation failed", liveMetricsRes.error.message); } else { this.liveMetrics.start(); } if (this.cfg.interpreterCheckIntervalMs > 0) { this.timer = setInterval(() => { this.tick(); }, this.cfg.interpreterCheckIntervalMs); } } stop() { this.stopping = true; if (this.timer) clearInterval(this.timer); this.timer = null; this.pool.stop(); this.liveMetrics.stop(); for (const j of this.journals.values()) j.stop(); this.journals.clear(); this.fineLagCoarseOnlyByStream.clear(); this.touchModeByStream.clear(); this.fineTokenBucketsByStream.clear(); this.hotFineByStream.clear(); this.lagSourceOffsetsByStream.clear(); this.restrictedTemplateBucketStateByStream.clear(); this.runtimeTotalsByStream.clear(); this.zeroRowBacklogStreakByStream.clear(); this.restartWorkerPoolRequested = false; this.lastWorkerPoolRestartAtMs = 0; } notify(stream) { this.dirty.add(stream); } async tick() { if (this.stopping) return; if (this.running) return; if (this.cfg.interpreterWorkers <= 0) return; this.running = true; try { const nowMs = Date.now(); const dirtyNow = new Set(this.dirty); this.dirty.clear(); const states = this.db.listStreamInterpreters(); if (states.length === 0) return; const stateByStream = new Map(states.map((s) => [s.stream, s])); const ordered = []; for (const s of dirtyNow) if (stateByStream.has(s)) ordered.push(s); for (const s of stateByStream.keys()) if (!dirtyNow.has(s)) ordered.push(s); const prioritized = this.prioritizeStreamsForProcessing(ordered, nowMs); const maxConcurrent = Math.max(1, this.cfg.interpreterWorkers); const tasks = []; if (prioritized.length > 0) { const total = prioritized.length; const start = this.streamScanCursor % total; for (let i = 0;i < total && tasks.length < maxConcurrent; i++) { if (this.stopping) break; const stream = prioritized[(start + i) % total]; if (this.failures.shouldSkip(stream)) continue; const st = stateByStream.get(stream); if (!st) continue; const p = this.processOne(stream, st.interpreted_through).catch((e) => { this.failures.recordFailure(stream); console.error("touch interpreter failed", stream, e); }); tasks.push(p); } this.streamScanCursor = (start + Math.max(1, tasks.length)) % total; } await Promise.all(tasks); if (this.restartWorkerPoolRequested) { this.restartWorkerPoolRequested = false; try { this.pool.restart(); this.lastWorkerPoolRestartAtMs = Date.now(); } catch (e) { console.error("touch interpreter worker-pool restart failed", e); } } for (const stream of stateByStream.keys()) { if (this.stopping) break; const srow = this.db.getStream(stream); if (!srow || this.db.isDeleted(srow)) continue; const interp = this.db.getStreamInterpreter(stream); if (!interp) continue; this.maybeGcBaseWal(stream, srow.uploaded_through, interp.interpreted_through); } const touchCfgByStream = new Map; let persistIntervalMin = Number.POSITIVE_INFINITY; for (const stream of stateByStream.keys()) { if (this.stopping) break; const regRes = this.registry.getRegistryResult(stream); if (Result15.isError(regRes)) { console.error("touch registry read failed", stream, regRes.error.message); continue; } const reg = regRes.value; if (!isTouchEnabled(reg.interpreter)) continue; const touchCfg = reg.interpreter.touch; touchCfgByStream.set(stream, touchCfg); const persistInterval = touchCfg.templates?.lastSeenPersistIntervalMs ?? 5 * 60 * 1000; if (persistInterval < persistIntervalMin) persistIntervalMin = persistInterval; } if (touchCfgByStream.size > 0 && Number.isFinite(persistIntervalMin)) { this.templates.flushLastSeen(nowMs, persistIntervalMin); } for (const [stream, touchCfg] of touchCfgByStream.entries()) { if (this.stopping) break; const gcInterval = touchCfg.templates?.gcIntervalMs ?? 60000; const last = this.lastTemplateGcMsByStream.get(stream) ?? 0; if (nowMs - last < gcInterval) continue; this.lastTemplateGcMsByStream.set(stream, nowMs); const retired = this.templates.gcRetireExpired(stream, nowMs); if (retired.retired.length > 0) { this.liveMetrics.emitLifecycle(retired.retired); } } } finally { this.running = false; } } async processOne(stream, interpretedThrough) { const srow = this.db.getStream(stream); if (!srow || this.db.isDeleted(srow)) { this.db.deleteStreamInterpreter(stream); return; } const next = srow.next_offset; if (next <= 0n) return; const fromOffset = interpretedThrough + 1n; const toOffset = next - 1n; if (fromOffset > toOffset) return; const regRes = this.registry.getRegistryResult(stream); if (Result15.isError(regRes)) { console.error("touch registry read failed", stream, regRes.error.message); this.db.deleteStreamInterpreter(stream); return; } const reg = regRes.value; if (!isTouchEnabled(reg.interpreter)) { this.db.deleteStreamInterpreter(stream); return; } const touchCfg = reg.interpreter.touch; const failProcessing = (message) => { this.failures.recordFailure(stream); this.liveMetrics.recordInterpreterError(stream, touchCfg); console.error("touch interpreter failed", stream, message); }; const nowMs = Date.now(); const hotFine = this.getHotFineSnapshot(stream, touchCfg, nowMs); const fineWaitersActive = hotFine?.fineWaitersActive ?? 0; const coarseWaitersActive = hotFine?.coarseWaitersActive ?? 0; const hasAnyWaiters = fineWaitersActive + coarseWaitersActive > 0; const hasFineDemand = fineWaitersActive > 0 || (hotFine?.broadFineWaitersActive ?? 0) > 0 || (hotFine?.hotKeyCount ?? 0) > 0 || (hotFine?.hotTemplateCount ?? 0) > 0; const lagAtStart = toOffset >= interpretedThrough ? toOffset - interpretedThrough : 0n; const suppressFineDueToLag = this.computeSuppressFineDueToLag(stream, touchCfg, lagAtStart, hasFineDemand); const j = this.getOrCreateJournal(stream, touchCfg); j.setCoalesceMs(this.computeAdaptiveCoalesceMs(touchCfg, lagAtStart, hasAnyWaiters)); const fineBudgetPerBatch = Math.max(0, Math.floor(touchCfg.fineTouchBudgetPerBatch ?? 2000)); const lagReservedFineBudgetPerBatch = Math.max(0, Math.floor(touchCfg.lagReservedFineTouchBudgetPerBatch ?? 200)); let fineBudget = !hasFineDemand ? 0 : suppressFineDueToLag ? lagReservedFineBudgetPerBatch : fineBudgetPerBatch; let tokenLimited = false; let refundFineTokens = null; if (fineBudget > 0) { const tokenGrant = this.reserveFineTokens(stream, touchCfg, fineBudget, nowMs); fineBudget = tokenGrant.granted; tokenLimited = tokenGrant.tokenLimited; refundFineTokens = tokenGrant.refund; } let emitFineTouches = hasFineDemand && fineBudget > 0; let fineGranularity = "key"; const batchStartMs = Date.now(); if (emitFineTouches && hotFine && hotFine.hotKeyCount === 0 && hotFine.hotTemplateCount === 0 && hotFine.broadFineWaitersActive === 0 && hotFine.keyFilteringEnabled && !hotFine.templateFilteringEnabled) { emitFineTouches = false; } if (emitFineTouches && suppressFineDueToLag) fineGranularity = "template"; if (fineGranularity !== "template") { this.restrictedTemplateBucketStateByStream.delete(stream); } const interpretMode = fineGranularity === "template" ? "hotTemplatesOnly" : "full"; const touchMode = !hasAnyWaiters ? "idle" : emitFineTouches ? suppressFineDueToLag ? "restricted" : "fine" : "coarseOnly"; this.touchModeByStream.set(stream, touchMode); const processRes = await this.pool.processResult({ stream, fromOffset, toOffset, interpreter: reg.interpreter, maxRows: Math.max(1, this.cfg.interpreterMaxBatchRows), maxBytes: Math.max(1, this.cfg.interpreterMaxBatchBytes), emitFineTouches, fineTouchBudget: emitFineTouches ? fineBudget : 0, fineGranularity, interpretMode, filterHotTemplates: !!(hotFine && hotFine.templateFilteringEnabled), hotTemplateIds: hotFine?.hotTemplateIdsForWorker ?? null }); if (Result15.isError(processRes)) { failProcessing(processRes.error.message); return; } const res = processRes.value; if (res.stats.rowsRead === 0 && toOffset >= fromOffset && this.rangeLikelyHasRows(stream, fromOffset, toOffset)) { const nextStreak = (this.zeroRowBacklogStreakByStream.get(stream) ?? 0) + 1; this.zeroRowBacklogStreakByStream.set(stream, nextStreak); if (nextStreak >= 5) { const now = Date.now(); if (now - this.lastWorkerPoolRestartAtMs >= 30000) { this.restartWorkerPoolRequested = true; console.error("touch interpreter produced zero-row batch despite WAL backlog; scheduling worker-pool restart", stream, `from=${fromOffset.toString()}`, `to=${toOffset.toString()}`); } } } else { this.zeroRowBacklogStreakByStream.delete(stream); } if (refundFineTokens) { refundFineTokens(Math.max(0, res.stats.templateTouchesEmitted ?? 0)); } const batchDurationMs = Math.max(0, Date.now() - batchStartMs); let touches = res.touches; const fineDroppedDueToBudget = Math.max(0, res.stats.fineTouchesDroppedDueToBudget ?? 0); let fineSkippedColdKey = 0; let fineSkippedTemplateBucket = 0; if (hotFine && hotFine.keyFilteringEnabled && fineGranularity !== "template") { const keyActiveSet = hotFine.hotKeyActiveSet; const keyGraceSet = hotFine.hotKeyGraceSet; const keyCount = (keyActiveSet?.size ?? 0) + (keyGraceSet?.size ?? 0); if (keyCount === 0) { for (const t of touches) if (t.kind === "template") fineSkippedColdKey += 1; touches = touches.filter((t) => t.kind === "table"); } else { const filtered = []; for (const t of touches) { if (t.kind !== "template") { filtered.push(t); continue; } const keyId = t.keyId >>> 0; if (keyActiveSet && keyActiveSet.has(keyId) || keyGraceSet && keyGraceSet.has(keyId)) { filtered.push(t); } else fineSkippedColdKey += 1; } touches = filtered; } } if (fineGranularity === "template" && touches.length > 0) { const coalesced = this.coalesceRestrictedTemplateTouches(stream, touchCfg, touches); touches = coalesced.touches; fineSkippedTemplateBucket = coalesced.dropped; } if (touches.length > 0) { const j2 = this.getOrCreateJournal(stream, touchCfg); for (const t of touches) { let sourceOffsetSeq; try { sourceOffsetSeq = BigInt(t.watermark); } catch { sourceOffsetSeq = undefined; } j2.touch(t.keyId >>> 0, sourceOffsetSeq); } } try { const lag = toOffset >= res.processedThrough ? toOffset - res.processedThrough : 0n; const lagNum = lag > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(lag); const effectiveLag = hasFineDemand ? lagNum : 0; this.lagSourceOffsetsByStream.set(stream, effectiveLag); const maxSourceTsMs = Number(res.stats.maxSourceTsMs ?? 0); const commitLagMs = maxSourceTsMs > 0 ? Math.max(0, Date.now() - maxSourceTsMs) : undefined; this.liveMetrics.recordInterpreterBatch({ stream, touchCfg, rowsRead: res.stats.rowsRead, changes: res.stats.changes, touches: touches.map((t) => ({ keyId: t.keyId >>> 0, kind: t.kind })), lagSourceOffsets: effectiveLag, commitLagMs, fineTouchesDroppedDueToBudget: fineDroppedDueToBudget, fineTouchesSkippedColdTemplate: Math.max(0, res.stats.fineTouchesSkippedColdTemplate ?? 0), fineTouchesSkippedColdKey: fineSkippedColdKey, fineTouchesSkippedTemplateBucket: fineSkippedTemplateBucket, fineTouchesSuppressedDueToLag: suppressFineDueToLag, fineTouchesSuppressedDueToLagMs: suppressFineDueToLag ? batchDurationMs : 0, fineTouchesSuppressedDueToBudget: !!res.stats.fineTouchesSuppressedDueToBudget || tokenLimited, touchMode, hotFineKeys: hotFine?.hotKeyCount ?? 0, hotTemplates: hotFine?.hotTemplateCount ?? 0, hotFineKeysActive: hotFine?.hotKeyActiveCount ?? 0, hotFineKeysGrace: hotFine?.hotKeyGraceCount ?? 0, hotTemplatesActive: hotFine?.hotTemplateActiveCount ?? 0, hotTemplatesGrace: hotFine?.hotTemplateGraceCount ?? 0, fineWaitersActive, coarseWaitersActive, broadFineWaitersActive: hotFine?.broadFineWaitersActive ?? 0, scannedButEmitted0: res.stats.rowsRead > 0 && touches.length === 0, noInterestFastForward: false, interpretedThroughDelta: res.processedThrough >= interpretedThrough ? Number(res.processedThrough - interpretedThrough > BigInt(Number.MAX_SAFE_INTEGER) ? BigInt(Number.MAX_SAFE_INTEGER) : res.processedThrough - interpretedThrough) : 0, touchesEmittedDelta: touches.length }); } catch {} const interpretedDelta = res.processedThrough >= interpretedThrough ? Number(res.processedThrough - interpretedThrough > BigInt(Number.MAX_SAFE_INTEGER) ? BigInt(Number.MAX_SAFE_INTEGER) : res.processedThrough - interpretedThrough) : 0; const totals = this.getOrCreateRuntimeTotals(stream); totals.scanBatchesTotal += 1; totals.scanRowsTotal += Math.max(0, res.stats.rowsRead); if (res.stats.rowsRead > 0 && touches.length === 0) totals.scannedButEmitted0BatchesTotal += 1; totals.interpretedThroughDeltaTotal += interpretedDelta; totals.touchesEmittedTotal += touches.length; let tableTouches = 0; let templateTouches = 0; for (const t of touches) { if (t.kind === "table") tableTouches += 1; else templateTouches += 1; } totals.touchesTableTotal += tableTouches; totals.touchesTemplateTotal += templateTouches; totals.fineTouchesDroppedDueToBudgetTotal += fineDroppedDueToBudget; totals.fineTouchesSkippedColdTemplateTotal += Math.max(0, res.stats.fineTouchesSkippedColdTemplate ?? 0); totals.fineTouchesSkippedColdKeyTotal += fineSkippedColdKey; totals.fineTouchesSkippedTemplateBucketTotal += fineSkippedTemplateBucket; this.db.updateStreamInterpreterThrough(stream, res.processedThrough); if (res.processedThrough < toOffset) this.dirty.add(stream); this.failures.recordSuccess(stream); } maybeGcBaseWal(stream, uploadedThrough, interpretedThrough) { const gcTargetThrough = interpretedThrough < uploadedThrough ? interpretedThrough : uploadedThrough; if (gcTargetThrough < 0n) return; const now = Date.now(); const last = this.lastBaseWalGc.get(stream) ?? { atMs: 0, through: -1n }; if (now - last.atMs < BASE_WAL_GC_INTERVAL_MS) return; if (gcTargetThrough <= last.through) { this.lastBaseWalGc.set(stream, { atMs: now, through: last.through }); return; } const chunk = BigInt(BASE_WAL_GC_CHUNK_OFFSETS2); const maxThroughThisSweep = chunk > 0n ? last.through + chunk : gcTargetThrough; const gcThrough = gcTargetThrough > maxThroughThisSweep ? maxThroughThisSweep : gcTargetThrough; try { const start = Date.now(); const res = this.db.deleteWalThrough(stream, gcThrough); const durationMs = Date.now() - start; if (res.deletedRows > 0 || res.deletedBytes > 0) { this.liveMetrics.recordBaseWalGc(stream, { deletedRows: res.deletedRows, deletedBytes: res.deletedBytes, durationMs }); } this.lastBaseWalGc.set(stream, { atMs: now, through: gcThrough }); } catch (e) { console.error("base WAL gc failed", stream, e); this.lastBaseWalGc.set(stream, { atMs: now, through: last.through }); } } seedInterpretersFromRegistry() { try { const rows = this.db.db.query(`SELECT stream, schema_json FROM schemas;`).all(); for (const row of rows) { const stream = String(row.stream); const json = String(row.schema_json ?? ""); let raw; try { raw = JSON.parse(json); } catch { continue; } const enabled = !!raw?.interpreter?.touch?.enabled; if (enabled) this.db.ensureStreamInterpreter(stream); else this.db.deleteStreamInterpreter(stream); } } catch {} } activateTemplates(args) { const nowMs = Date.now(); const limits = { maxActiveTemplatesPerStream: args.touchCfg.templates?.maxActiveTemplatesPerStream ?? 2048, maxActiveTemplatesPerEntity: args.touchCfg.templates?.maxActiveTemplatesPerEntity ?? 256, activationRateLimitPerMinute: args.touchCfg.templates?.activationRateLimitPerMinute ?? 100 }; const res = this.templates.activate({ stream: args.stream, activeFromTouchOffset: args.activeFromTouchOffset, baseStreamNextOffset: args.baseStreamNextOffset, templates: args.templates, inactivityTtlMs: args.inactivityTtlMs, limits, nowMs }); const deniedRate = res.denied.filter((d) => d.reason === "rate_limited").length; if (deniedRate > 0) this.liveMetrics.recordActivationDenied(args.stream, args.touchCfg, deniedRate); if (res.lifecycle.length > 0) this.liveMetrics.emitLifecycle(res.lifecycle); return { activated: res.activated, denied: res.denied }; } heartbeatTemplates(args) { const nowMs = Date.now(); this.templates.heartbeat(args.stream, args.templateIdsUsed, nowMs); const persistInterval = args.touchCfg.templates?.lastSeenPersistIntervalMs ?? 5 * 60 * 1000; this.templates.flushLastSeen(nowMs, persistInterval); } beginHotWaitInterest(args) { const nowMs = Date.now(); const limits = this.getHotFineLimits(args.touchCfg); const state = this.getOrCreateHotFineState(args.stream); const isFine = args.interestMode === "fine"; if (isFine) state.fineWaitersActive += 1; else state.coarseWaitersActive += 1; const trackedKeyIds = []; const trackedTemplateIds = []; const broad = isFine && args.keyIds.length > HOT_INTEREST_MAX_KEYS; if (!isFine) {} else if (broad) { state.broadFineWaitersActive += 1; } else { const uniqueKeys = new Set(args.keyIds.map((raw) => Number(raw) >>> 0)); for (const keyId of uniqueKeys) { if (this.acquireHotKey(state, keyId, limits.maxKeys)) trackedKeyIds.push(keyId); } } if (isFine) { const uniqueTemplates = new Set; for (const raw of args.templateIdsUsed) { const templateId = String(raw).trim(); if (!/^[0-9a-f]{16}$/.test(templateId)) continue; uniqueTemplates.add(templateId); } for (const templateId of uniqueTemplates) { if (this.acquireHotTemplate(state, templateId, limits.maxTemplates)) trackedTemplateIds.push(templateId); } } this.sweepHotFineState(args.stream, args.touchCfg, nowMs, true); let released = false; return () => { if (released) return; released = true; const st = this.hotFineByStream.get(args.stream); if (!st) return; const releaseNowMs = Date.now(); if (isFine) st.fineWaitersActive = Math.max(0, st.fineWaitersActive - 1); else st.coarseWaitersActive = Math.max(0, st.coarseWaitersActive - 1); if (broad) st.broadFineWaitersActive = Math.max(0, st.broadFineWaitersActive - 1); for (const keyId of trackedKeyIds) { this.releaseHotKey(st, keyId, releaseNowMs, limits.keyGraceMs, limits.maxKeys); } for (const templateId of trackedTemplateIds) { this.releaseHotTemplate(st, templateId, releaseNowMs, limits.templateGraceMs, limits.maxTemplates); } this.sweepHotFineState(args.stream, args.touchCfg, releaseNowMs, true); }; } getTouchRuntimeSnapshot(args) { const nowMs = Date.now(); const hot = this.getHotFineSnapshot(args.stream, args.touchCfg, nowMs); const totals = this.getOrCreateRuntimeTotals(args.stream); const journal = this.journals.get(args.stream) ?? null; const journalTotals = journal?.getTotalStats(); return { lagSourceOffsets: this.lagSourceOffsetsByStream.get(args.stream) ?? 0, touchMode: this.touchModeByStream.get(args.stream) ?? (this.fineLagCoarseOnlyByStream.get(args.stream) ? "coarseOnly" : "fine"), hotFineKeys: hot?.hotKeyCount ?? 0, hotTemplates: hot?.hotTemplateCount ?? 0, hotFineKeysActive: hot?.hotKeyActiveCount ?? 0, hotFineKeysGrace: hot?.hotKeyGraceCount ?? 0, hotTemplatesActive: hot?.hotTemplateActiveCount ?? 0, hotTemplatesGrace: hot?.hotTemplateGraceCount ?? 0, fineWaitersActive: hot?.fineWaitersActive ?? 0, coarseWaitersActive: hot?.coarseWaitersActive ?? 0, broadFineWaitersActive: hot?.broadFineWaitersActive ?? 0, hotKeyFilteringEnabled: hot?.keyFilteringEnabled ?? false, hotTemplateFilteringEnabled: hot?.templateFilteringEnabled ?? false, scanRowsTotal: totals.scanRowsTotal, scanBatchesTotal: totals.scanBatchesTotal, scannedButEmitted0BatchesTotal: totals.scannedButEmitted0BatchesTotal, interpretedThroughDeltaTotal: totals.interpretedThroughDeltaTotal, touchesEmittedTotal: totals.touchesEmittedTotal, touchesTableTotal: totals.touchesTableTotal, touchesTemplateTotal: totals.touchesTemplateTotal, fineTouchesDroppedDueToBudgetTotal: totals.fineTouchesDroppedDueToBudgetTotal, fineTouchesSkippedColdTemplateTotal: totals.fineTouchesSkippedColdTemplateTotal, fineTouchesSkippedColdKeyTotal: totals.fineTouchesSkippedColdKeyTotal, fineTouchesSkippedTemplateBucketTotal: totals.fineTouchesSkippedTemplateBucketTotal, waitTouchedTotal: totals.waitTouchedTotal, waitTimeoutTotal: totals.waitTimeoutTotal, waitStaleTotal: totals.waitStaleTotal, journalFlushesTotal: journalTotals?.flushes ?? 0, journalNotifyWakeupsTotal: journalTotals?.notifyWakeups ?? 0, journalNotifyWakeMsTotal: journalTotals?.notifyWakeMsSum ?? 0, journalNotifyWakeMsMax: journalTotals?.notifyWakeMsMax ?? 0, journalTimeoutsFiredTotal: journalTotals?.timeoutsFired ?? 0, journalTimeoutSweepMsTotal: journalTotals?.timeoutSweepMsSum ?? 0 }; } recordWaitMetrics(args) { this.liveMetrics.recordWait(args.stream, args.touchCfg, args.keysCount, args.outcome, args.latencyMs); const totals = this.getOrCreateRuntimeTotals(args.stream); if (args.outcome === "touched") totals.waitTouchedTotal += 1; else if (args.outcome === "timeout") totals.waitTimeoutTotal += 1; else totals.waitStaleTotal += 1; } resolveTemplateEntitiesForWait(args) { const ids = Array.from(new Set(args.templateIdsUsed.map((x) => String(x).trim()).filter((x) => /^[0-9a-f]{16}$/.test(x)))); if (ids.length === 0) return []; const entities = new Set; const chunkSize = 200; for (let i = 0;i < ids.length; i += chunkSize) { const chunk = ids.slice(i, i + chunkSize); const placeholders = chunk.map(() => "?").join(","); const rows = this.db.db.query(`SELECT DISTINCT entity FROM live_templates WHERE stream=? AND state='active' AND template_id IN (${placeholders});`).all(args.stream, ...chunk); for (const row of rows) { const entity = String(row?.entity ?? "").trim(); if (entity !== "") entities.add(entity); } } return Array.from(entities); } getOrCreateJournal(stream, touchCfg) { const existing = this.journals.get(stream); if (existing) return existing; const mem = touchCfg.memory ?? {}; const j = new TouchJournal({ bucketMs: mem.bucketMs ?? 100, filterPow2: mem.filterPow2 ?? 22, k: mem.k ?? 4, pendingMaxKeys: mem.pendingMaxKeys ?? 1e5, keyIndexMaxKeys: mem.keyIndexMaxKeys ?? 32 }); this.journals.set(stream, j); return j; } computeAdaptiveCoalesceMs(touchCfg, lagAtStart, hasAnyWaiters) { const maxCoalesceMs = Math.max(1, Math.floor(touchCfg.memory?.bucketMs ?? 100)); if (!hasAnyWaiters) return maxCoalesceMs; const lagNum = lagAtStart > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(lagAtStart); if (lagNum <= 0) return Math.min(maxCoalesceMs, 10); if (lagNum <= 5000) return Math.min(maxCoalesceMs, 50); return maxCoalesceMs; } getJournalIfExists(stream) { return this.journals.get(stream) ?? null; } computeSuppressFineDueToLag(stream, touchCfg, lagAtStart, hasFineDemand) { if (!hasFineDemand) { this.fineLagCoarseOnlyByStream.set(stream, false); return false; } const degradeRaw = Math.max(0, Math.floor(touchCfg.lagDegradeFineTouchesAtSourceOffsets ?? 5000)); if (degradeRaw <= 0) { this.fineLagCoarseOnlyByStream.set(stream, false); return false; } const recoverRaw = Math.max(0, Math.floor(touchCfg.lagRecoverFineTouchesAtSourceOffsets ?? 1000)); const recover = Math.min(degradeRaw, recoverRaw); const lag = lagAtStart > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(lagAtStart); const prev = this.fineLagCoarseOnlyByStream.get(stream) ?? false; let next = prev; if (!prev && lag >= degradeRaw) next = true; else if (prev && lag <= recover) next = false; this.fineLagCoarseOnlyByStream.set(stream, next); return next; } prioritizeStreamsForProcessing(ordered, nowMs) { if (ordered.length <= 1) return ordered; const hot = []; const cold = []; for (const stream of ordered) { let hasActiveWaiters = false; const regRes = this.registry.getRegistryResult(stream); if (Result15.isError(regRes)) { hasActiveWaiters = false; } else { const reg = regRes.value; if (isTouchEnabled(reg.interpreter)) { const snap = this.getHotFineSnapshot(stream, reg.interpreter.touch, nowMs); hasActiveWaiters = snap.fineWaitersActive + snap.coarseWaitersActive > 0; } } if (hasActiveWaiters) hot.push(stream); else cold.push(stream); } if (hot.length === 0) return ordered; return hot.concat(cold); } coalesceRestrictedTemplateTouches(stream, touchCfg, touches) { const bucketMs = Math.max(1, Math.floor(touchCfg.memory?.bucketMs ?? 100)); const bucketId = Math.floor(Date.now() / bucketMs); let state = this.restrictedTemplateBucketStateByStream.get(stream); if (!state || state.bucketId !== bucketId) { state = { bucketId, templateKeyIds: new Set }; this.restrictedTemplateBucketStateByStream.set(stream, state); } const out = []; let dropped = 0; for (const touch of touches) { if (touch.kind !== "template") { out.push(touch); continue; } const keyId = touch.keyId >>> 0; if (state.templateKeyIds.has(keyId)) { dropped += 1; continue; } state.templateKeyIds.add(keyId); out.push(touch); } return { touches: out, dropped }; } getHotFineSnapshot(stream, touchCfg, nowMs) { const state = this.sweepHotFineState(stream, touchCfg, nowMs, false); if (!state) { return { hotTemplateIdsForWorker: null, hotKeyActiveSet: null, hotKeyGraceSet: null, hotTemplateActiveCount: 0, hotTemplateGraceCount: 0, hotKeyActiveCount: 0, hotKeyGraceCount: 0, hotTemplateCount: 0, hotKeyCount: 0, fineWaitersActive: 0, coarseWaitersActive: 0, broadFineWaitersActive: 0, templateFilteringEnabled: false, keyFilteringEnabled: true }; } const hotTemplateActiveCount = state.templateActiveCountsById.size; const hotTemplateGraceCount = state.templateGraceExpiryMsById.size; const hotKeyActiveCount = state.keyActiveCountsById.size; const hotKeyGraceCount = state.keyGraceExpiryMsById.size; const hotTemplateCount = hotTemplateActiveCount + hotTemplateGraceCount; const hotKeyCount = hotKeyActiveCount + hotKeyGraceCount; const templateFilteringEnabled = !state.templatesOverCapacity && hotTemplateCount > 0; const keyFilteringEnabled = !state.keysOverCapacity && state.broadFineWaitersActive === 0; const hotTemplateIdsForWorker = templateFilteringEnabled ? Array.from(new Set([...state.templateActiveCountsById.keys(), ...state.templateGraceExpiryMsById.keys()])) : null; return { hotTemplateIdsForWorker, hotKeyActiveSet: keyFilteringEnabled ? state.keyActiveCountsById : null, hotKeyGraceSet: keyFilteringEnabled ? state.keyGraceExpiryMsById : null, hotTemplateActiveCount, hotTemplateGraceCount, hotKeyActiveCount, hotKeyGraceCount, hotTemplateCount, hotKeyCount, fineWaitersActive: state.fineWaitersActive, coarseWaitersActive: state.coarseWaitersActive, broadFineWaitersActive: state.broadFineWaitersActive, templateFilteringEnabled, keyFilteringEnabled }; } getOrCreateHotFineState(stream) { const existing = this.hotFineByStream.get(stream); if (existing) return existing; const created = { keyActiveCountsById: new Map, keyGraceExpiryMsById: new Map, templateActiveCountsById: new Map, templateGraceExpiryMsById: new Map, fineWaitersActive: 0, coarseWaitersActive: 0, broadFineWaitersActive: 0, nextSweepAtMs: 0, keysOverCapacity: false, templatesOverCapacity: false }; this.hotFineByStream.set(stream, created); return created; } sweepHotFineState(stream, touchCfg, nowMs, force) { const state = this.hotFineByStream.get(stream); if (!state) return null; if (!force && nowMs < state.nextSweepAtMs) return state; const limits = this.getHotFineLimits(touchCfg); for (const [k, exp] of state.keyGraceExpiryMsById.entries()) { if (exp <= nowMs) state.keyGraceExpiryMsById.delete(k); } for (const [tpl, exp] of state.templateGraceExpiryMsById.entries()) { if (exp <= nowMs) state.templateGraceExpiryMsById.delete(tpl); } if (state.keyActiveCountsById.size + state.keyGraceExpiryMsById.size < limits.maxKeys) state.keysOverCapacity = false; if (state.templateActiveCountsById.size + state.templateGraceExpiryMsById.size < limits.maxTemplates) state.templatesOverCapacity = false; if (state.keyActiveCountsById.size === 0 && state.keyGraceExpiryMsById.size === 0 && state.templateActiveCountsById.size === 0 && state.templateGraceExpiryMsById.size === 0 && state.fineWaitersActive <= 0 && state.coarseWaitersActive <= 0 && state.broadFineWaitersActive <= 0) { this.hotFineByStream.delete(stream); return null; } const sweepEveryMs = Math.max(250, Math.min(limits.keyGraceMs, limits.templateGraceMs, 2000)); state.nextSweepAtMs = nowMs + sweepEveryMs; return state; } getHotFineLimits(touchCfg) { const mem = touchCfg.memory ?? {}; return { keyGraceMs: Math.max(1, Math.floor(mem.hotKeyTtlMs ?? 1e4)), templateGraceMs: Math.max(1, Math.floor(mem.hotTemplateTtlMs ?? 1e4)), maxKeys: Math.max(1, Math.floor(mem.hotMaxKeys ?? 1e6)), maxTemplates: Math.max(1, Math.floor(mem.hotMaxTemplates ?? 4096)) }; } acquireHotKey(state, keyId, maxKeys) { const prev = state.keyActiveCountsById.get(keyId); if (prev != null) { state.keyActiveCountsById.set(keyId, prev + 1); state.keyGraceExpiryMsById.delete(keyId); return true; } if (state.keyActiveCountsById.size + state.keyGraceExpiryMsById.size >= maxKeys) { state.keysOverCapacity = true; return false; } state.keyActiveCountsById.set(keyId, 1); state.keyGraceExpiryMsById.delete(keyId); return true; } acquireHotTemplate(state, templateId, maxTemplates) { const prev = state.templateActiveCountsById.get(templateId); if (prev != null) { state.templateActiveCountsById.set(templateId, prev + 1); state.templateGraceExpiryMsById.delete(templateId); return true; } if (state.templateActiveCountsById.size + state.templateGraceExpiryMsById.size >= maxTemplates) { state.templatesOverCapacity = true; return false; } state.templateActiveCountsById.set(templateId, 1); state.templateGraceExpiryMsById.delete(templateId); return true; } releaseHotKey(state, keyId, nowMs, keyGraceMs, maxKeys) { const prev = state.keyActiveCountsById.get(keyId); if (prev == null) return; if (prev > 1) { state.keyActiveCountsById.set(keyId, prev - 1); return; } state.keyActiveCountsById.delete(keyId); if (keyGraceMs <= 0) { state.keyGraceExpiryMsById.delete(keyId); return; } if (state.keyActiveCountsById.size + state.keyGraceExpiryMsById.size >= maxKeys) { state.keysOverCapacity = true; return; } state.keyGraceExpiryMsById.set(keyId, nowMs + keyGraceMs); } releaseHotTemplate(state, templateId, nowMs, templateGraceMs, maxTemplates) { const prev = state.templateActiveCountsById.get(templateId); if (prev == null) return; if (prev > 1) { state.templateActiveCountsById.set(templateId, prev - 1); return; } state.templateActiveCountsById.delete(templateId); if (templateGraceMs <= 0) { state.templateGraceExpiryMsById.delete(templateId); return; } if (state.templateActiveCountsById.size + state.templateGraceExpiryMsById.size >= maxTemplates) { state.templatesOverCapacity = true; return; } state.templateGraceExpiryMsById.set(templateId, nowMs + templateGraceMs); } reserveFineTokens(stream, touchCfg, wanted, nowMs) { const rate = Math.max(0, Math.floor(touchCfg.fineTokensPerSecond ?? 200000)); const burst = Math.max(0, Math.floor(touchCfg.fineBurstTokens ?? 400000)); if (wanted <= 0) return { granted: 0, tokenLimited: false, refund: () => {} }; if (rate <= 0 || burst <= 0) return { granted: 0, tokenLimited: true, refund: () => {} }; const b = this.fineTokenBucketsByStream.get(stream) ?? { tokens: burst, lastRefillMs: nowMs }; const elapsedMs = Math.max(0, nowMs - b.lastRefillMs); if (elapsedMs > 0) { const refill = elapsedMs * rate / 1000; b.tokens = Math.min(burst, b.tokens + refill); b.lastRefillMs = nowMs; } const granted = Math.max(0, Math.min(wanted, Math.floor(b.tokens))); b.tokens = Math.max(0, b.tokens - granted); this.fineTokenBucketsByStream.set(stream, b); return { granted, tokenLimited: granted < wanted, refund: (used) => { const u = Math.max(0, Math.floor(used)); if (u >= granted) return; const addBack = granted - u; const cur = this.fineTokenBucketsByStream.get(stream); if (!cur) return; cur.tokens = Math.min(burst, cur.tokens + addBack); this.fineTokenBucketsByStream.set(stream, cur); } }; } getOrCreateRuntimeTotals(stream) { const existing = this.runtimeTotalsByStream.get(stream); if (existing) return existing; const created = { scanRowsTotal: 0, scanBatchesTotal: 0, scannedButEmitted0BatchesTotal: 0, interpretedThroughDeltaTotal: 0, touchesEmittedTotal: 0, touchesTableTotal: 0, touchesTemplateTotal: 0, fineTouchesDroppedDueToBudgetTotal: 0, fineTouchesSkippedColdTemplateTotal: 0, fineTouchesSkippedColdKeyTotal: 0, fineTouchesSkippedTemplateBucketTotal: 0, waitTouchedTotal: 0, waitTimeoutTotal: 0, waitStaleTotal: 0 }; this.runtimeTotalsByStream.set(stream, created); return created; } rangeLikelyHasRows(stream, fromOffset, toOffset) { try { const it = this.db.iterWalRange(stream, fromOffset, toOffset); const first = it.next(); return !first.done; } catch { return false; } } } class FailureTracker { cache; constructor(maxEntries) { this.cache = new LruCache(maxEntries); } shouldSkip(stream) { const item = this.cache.get(stream); if (!item) return false; if (Date.now() >= item.untilMs) { this.cache.delete(stream); return false; } return true; } recordFailure(stream) { const now = Date.now(); const item = this.cache.get(stream) ?? { attempts: 0, untilMs: now }; item.attempts += 1; const backoff = Math.min(60000, 500 * 2 ** (item.attempts - 1)); item.untilMs = now + backoff; this.cache.set(stream, item); } recordSuccess(stream) { this.cache.delete(stream); } } // src/touch/touch_key_id.ts import { Result as Result16 } from "better-result"; function touchKeyIdFromRoutingKeyResult(key) { const s = key.trim().toLowerCase(); if (/^[0-9a-f]{16}$/.test(s)) { return Result16.ok(Number.parseInt(s.slice(8), 16) >>> 0); } return xxh32Result(s); } // src/app_core.ts import { Result as Result17 } from "better-result"; function withNosniff(headers = {}) { return { "x-content-type-options": "nosniff", ...headers }; } function json(status, body, headers = {}) { return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store", ...withNosniff(headers) } }); } function internalError(message = "internal server error") { return json(500, { error: { code: "internal", message } }); } function badRequest(msg) { return json(400, { error: { code: "bad_request", message: msg } }); } function notFound(msg = "not_found") { return json(404, { error: { code: "not_found", message: msg } }); } function readerErrorResponse(err) { if (err.kind === "not_found") return notFound(); if (err.kind === "gone") return notFound("stream expired"); if (err.kind === "internal") return internalError(); return badRequest(err.message); } function schemaMutationErrorResponse(err) { if (err.kind === "version_mismatch") return conflict(err.message); return badRequest(err.message); } function schemaReadErrorResponse(_err) { return internalError(); } function conflict(msg, headers = {}) { return json(409, { error: { code: "conflict", message: msg } }, headers); } function tooLarge(msg) { return json(413, { error: { code: "payload_too_large", message: msg } }); } function normalizeContentType(value) { if (!value) return null; const base = value.split(";")[0]?.trim().toLowerCase(); return base ? base : null; } function isJsonContentType(value) { return normalizeContentType(value) === "application/json"; } function isTextContentType(value) { const norm = normalizeContentType(value); return norm === "application/json" || norm != null && norm.startsWith("text/"); } function parseStreamClosedHeader(value) { return value != null && value.trim().toLowerCase() === "true"; } function parseStreamSeqHeader(value) { if (value == null) return Result17.ok(null); const v = value.trim(); if (v.length === 0) return Result17.err({ message: "invalid Stream-Seq" }); return Result17.ok(v); } function parseStreamTtlSeconds(value) { const s = value.trim(); if (/^(0|[1-9][0-9]*)$/.test(s)) return Result17.ok(Number(s)); if (/^(0|[1-9][0-9]*)(ms|s|m|h|d)$/.test(s)) { const msRes = parseDurationMsResult(s); if (Result17.isError(msRes)) return Result17.err({ message: msRes.error.message }); const ms = msRes.value; if (ms % 1000 !== 0) return Result17.err({ message: "invalid Stream-TTL" }); return Result17.ok(Math.floor(ms / 1000)); } return Result17.err({ message: "invalid Stream-TTL" }); } function parseNonNegativeInt(value) { if (!/^[0-9]+$/.test(value)) return null; const n = Number(value); if (!Number.isFinite(n)) return null; return n; } function splitSseLines(data) { if (data === "") return [""]; return data.split(/\r\n|\r|\n/); } function encodeSseEvent(eventType, data) { const lines = splitSseLines(data); let out = `event: ${eventType} `; for (const line of lines) { out += `data:${line} `; } out += ` `; return out; } function computeCursor(nowMs, provided) { let cursor = Math.floor(nowMs / 1000); if (provided && /^[0-9]+$/.test(provided)) { const n = Number(provided); if (Number.isFinite(n) && n >= cursor) cursor = n + 1; } return String(cursor); } function concatPayloads(parts) { let total = 0; for (const p of parts) total += p.byteLength; const out = new Uint8Array(total); let off = 0; for (const p of parts) { out.set(p, off); off += p.byteLength; } return out; } function keyBytesFromString(s) { if (s == null) return null; return new TextEncoder().encode(s); } function extractRoutingKey(reg, value) { if (!reg.routingKey) return Result17.ok(null); const { jsonPointer, required } = reg.routingKey; const resolvedRes = resolvePointerResult(value, jsonPointer); if (Result17.isError(resolvedRes)) return Result17.err({ message: resolvedRes.error.message }); const resolved = resolvedRes.value; if (!resolved.exists) { if (required) return Result17.err({ message: "routing key missing" }); return Result17.ok(null); } if (typeof resolved.value !== "string") return Result17.err({ message: "routing key must be string" }); return Result17.ok(keyBytesFromString(resolved.value)); } function schemaVersionForOffset(reg, offset) { if (!reg.boundaries || reg.boundaries.length === 0) return 0; const off = Number(offset); let version = 0; for (const b of reg.boundaries) { if (b.offset <= off) version = b.version; else break; } return version; } function createAppCore(cfg, opts) { mkdirSync(cfg.rootDir, { recursive: true }); cleanupTempSegments(cfg.rootDir); const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes }); db.resetSegmentInProgress(); const stats = opts.stats; const backpressure = cfg.localBacklogMaxBytes > 0 ? new BackpressureGate(cfg.localBacklogMaxBytes, db.sumPendingBytes() + db.sumPendingSegmentBytes()) : undefined; const memory = new MemoryGuard(cfg.memoryLimitBytes, { onSample: (rss, overLimit) => { metrics.record("process.rss.bytes", rss, "bytes"); if (overLimit) metrics.record("process.rss.over_limit", 1, "count"); }, heapSnapshotPath: `${cfg.rootDir}/heap.heapsnapshot` }); memory.start(); const metrics = new Metrics; const ingest = new IngestQueue(cfg, db, stats, backpressure, memory, metrics); const notifier = new StreamNotifier; const registry = new SchemaRegistryStore(db); const touch = new TouchInterpreterManager(cfg, db, ingest, notifier, registry, backpressure); const runtime = opts.createRuntime({ config: cfg, db, ingest, notifier, registry, touch, stats, backpressure, memory, metrics }); const { store, reader, segmenter, uploader, indexer, uploadSchemaRegistry } = runtime; const metricsEmitter = new MetricsEmitter(metrics, ingest, cfg.metricsFlushIntervalMs); const expirySweeper = new ExpirySweeper(cfg, db); db.ensureStream("__stream_metrics__", { contentType: "application/json" }); runtime.start(); metricsEmitter.start(); expirySweeper.start(); touch.start(); const buildJsonRows = (stream, bodyBytes, routingKeyHeader, allowEmptyArray) => { const regRes = registry.getRegistryResult(stream); if (Result17.isError(regRes)) { return Result17.err({ status: 500, message: regRes.error.message }); } const reg = regRes.value; const text = new TextDecoder().decode(bodyBytes); let arr; try { arr = JSON.parse(text); } catch { return Result17.err({ status: 400, message: "invalid JSON" }); } if (!Array.isArray(arr)) arr = [arr]; if (arr.length === 0 && !allowEmptyArray) return Result17.err({ status: 400, message: "empty JSON array" }); if (reg.routingKey && routingKeyHeader) { return Result17.err({ status: 400, message: "Stream-Key not allowed when routingKey is configured" }); } const validator = reg.currentVersion > 0 ? registry.getValidatorForVersion(reg, reg.currentVersion) : null; if (reg.currentVersion > 0 && !validator) { return Result17.err({ status: 500, message: "schema validator missing" }); } const rows = []; for (const v of arr) { if (validator && !validator(v)) { const msg = validator.errors ? validator.errors.map((e) => e.message).join("; ") : "schema validation failed"; return Result17.err({ status: 400, message: msg }); } const rkRes = reg.routingKey ? extractRoutingKey(reg, v) : Result17.ok(keyBytesFromString(routingKeyHeader)); if (Result17.isError(rkRes)) return Result17.err({ status: 400, message: rkRes.error.message }); rows.push({ routingKey: rkRes.value, contentType: "application/json", payload: new TextEncoder().encode(JSON.stringify(v)) }); } return Result17.ok({ rows }); }; const buildAppendRowsResult = (stream, bodyBytes, contentType, routingKeyHeader, allowEmptyJsonArray) => { if (isJsonContentType(contentType)) { return buildJsonRows(stream, bodyBytes, routingKeyHeader, allowEmptyJsonArray); } const regRes = registry.getRegistryResult(stream); if (Result17.isError(regRes)) return Result17.err({ status: 500, message: regRes.error.message }); const reg = regRes.value; if (reg.currentVersion > 0) return Result17.err({ status: 400, message: "stream requires JSON" }); return Result17.ok({ rows: [ { routingKey: keyBytesFromString(routingKeyHeader), contentType, payload: bodyBytes } ] }); }; const enqueueAppend = (args) => ingest.append({ stream: args.stream, baseAppendMs: args.baseAppendMs, rows: args.rows, contentType: args.contentType, streamSeq: args.streamSeq, producer: args.producer, close: args.close }); const recordAppendOutcome = (args) => { if (args.appendedRows > 0) { metrics.recordAppend(args.metricsBytes, args.appendedRows); notifier.notify(args.stream, args.lastOffset); touch.notify(args.stream); } if (stats) { if (args.touched) stats.recordStreamTouched(args.stream); if (args.appendedRows > 0) stats.recordIngested(args.ingestedBytes); } if (args.closed) notifier.notifyClose(args.stream); }; const decodeJsonRecords = (stream, records) => { const regRes = registry.getRegistryResult(stream); if (Result17.isError(regRes)) return Result17.err({ status: 500, message: regRes.error.message }); const reg = regRes.value; const values = []; for (const r of records) { try { const s = new TextDecoder().decode(r.payload); let value = JSON.parse(s); if (reg.currentVersion > 0) { const version = schemaVersionForOffset(reg, r.offset); if (version < reg.currentVersion) { const chainRes = registry.getLensChainResult(reg, version, reg.currentVersion); if (Result17.isError(chainRes)) return Result17.err({ status: 500, message: chainRes.error.message }); const chain = chainRes.value; const transformedRes = applyLensChainResult(chain, value); if (Result17.isError(transformedRes)) return Result17.err({ status: 400, message: transformedRes.error.message }); value = transformedRes.value; } } values.push(value); } catch (e) { return Result17.err({ status: 400, message: String(e?.message ?? e) }); } } return Result17.ok({ values }); }; let closing = false; const fetch2 = async (req) => { if (closing) { return json(503, { error: { code: "unavailable", message: "server shutting down" } }); } try { let url; try { url = new URL(req.url, "http://localhost"); } catch { return badRequest("invalid url"); } const path = url.pathname; if (path === "/health") { return json(200, { ok: true }); } if (path === "/metrics") { return json(200, metrics.snapshot()); } const rejectIfMemoryLimited = () => { if (!memory || memory.shouldAllow()) return null; memory.maybeGc("memory limit"); memory.maybeHeapSnapshot("memory limit"); metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "memory" }); return json(429, { error: { code: "overloaded", message: "ingest queue full" } }); }; if (req.method === "GET" && path === "/v1/streams") { const limit = Number(url.searchParams.get("limit") ?? "100"); const offset = Number(url.searchParams.get("offset") ?? "0"); const rows = db.listStreams(Math.max(0, Math.min(limit, 1000)), Math.max(0, offset)); const out = rows.map((r) => ({ name: r.stream, created_at: new Date(Number(r.created_at_ms)).toISOString(), expires_at: r.expires_at_ms == null ? null : new Date(Number(r.expires_at_ms)).toISOString(), epoch: r.epoch, next_offset: r.next_offset.toString(), sealed_through: r.sealed_through.toString(), uploaded_through: r.uploaded_through.toString() })); return json(200, out); } const streamPrefix = "/v1/stream/"; if (path.startsWith(streamPrefix)) { const rawRest = path.slice(streamPrefix.length); const rest = rawRest.replace(/\/+$/, ""); if (rest.length === 0) return badRequest("missing stream name"); const segments = rest.split("/"); let isSchema = false; let pathKeyParam = null; let touchMode = null; if (segments[segments.length - 1] === "_schema") { isSchema = true; segments.pop(); } else if (segments.length >= 3 && segments[segments.length - 3] === "touch" && segments[segments.length - 2] === "templates" && segments[segments.length - 1] === "activate") { touchMode = { kind: "templates_activate" }; segments.splice(segments.length - 3, 3); } else if (segments.length >= 2 && segments[segments.length - 2] === "touch" && segments[segments.length - 1] === "meta") { touchMode = { kind: "meta" }; segments.splice(segments.length - 2, 2); } else if (segments.length >= 2 && segments[segments.length - 2] === "touch" && segments[segments.length - 1] === "wait") { touchMode = { kind: "wait" }; segments.splice(segments.length - 2, 2); } else if (segments.length >= 2 && segments[segments.length - 2] === "pk") { pathKeyParam = decodeURIComponent(segments[segments.length - 1]); segments.splice(segments.length - 2, 2); } const streamPart = segments.join("/"); if (streamPart.length === 0) return badRequest("missing stream name"); const stream = decodeURIComponent(streamPart); if (isSchema) { const srow = db.getStream(stream); if (!srow || db.isDeleted(srow)) return notFound(); if (srow.expires_at_ms != null && db.nowMs() > srow.expires_at_ms) return notFound("stream expired"); if (req.method === "GET") { const regRes = registry.getRegistryResult(stream); if (Result17.isError(regRes)) return schemaReadErrorResponse(regRes.error); return json(200, regRes.value); } if (req.method === "POST") { let body; try { body = await req.json(); } catch { return badRequest("schema update must be valid JSON"); } let update = body; const isSchemaObject = update && (update.schema === true || update.schema === false || typeof update.schema === "object" && update.schema !== null && !Array.isArray(update.schema)); if (!isSchemaObject && update && typeof update === "object" && update.schemas && typeof update.schemas === "object") { const versions = Object.keys(update.schemas).map((v) => Number(v)).filter((v) => Number.isFinite(v) && v >= 0); const currentVersion = typeof update.currentVersion === "number" && Number.isFinite(update.currentVersion) ? update.currentVersion : versions.length > 0 ? Math.max(...versions) : null; if (currentVersion != null) { const schema = update.schemas[String(currentVersion)]; const lens = update.lens ?? (update.lenses && typeof update.lenses === "object" ? update.lenses[String(currentVersion - 1)] : undefined); update = { schema, lens, routingKey: update.routingKey, interpreter: update.interpreter }; } } if (update && typeof update === "object") { if (update.schema === null) { delete update.schema; } if (update.routingKey === undefined) { const raw = update; const candidate = raw.routing_key ?? raw.routingKeyPointer ?? raw.routing_key_pointer ?? raw.routingKey; if (typeof candidate === "string") { update.routingKey = { jsonPointer: candidate, required: true }; } else if (candidate && typeof candidate === "object") { const jsonPointer = candidate.jsonPointer ?? candidate.json_pointer; if (typeof jsonPointer === "string") { update.routingKey = { jsonPointer, required: typeof candidate.required === "boolean" ? candidate.required : true }; } } } else if (update.routingKey && typeof update.routingKey === "object") { const rk = update.routingKey; if (rk.jsonPointer === undefined && typeof rk.json_pointer === "string") { update.routingKey = { jsonPointer: rk.json_pointer, required: typeof rk.required === "boolean" ? rk.required : true }; } } } if (update.schema === undefined && update.routingKey !== undefined && update.interpreter === undefined) { const regRes2 = registry.updateRoutingKeyResult(stream, update.routingKey ?? null); if (Result17.isError(regRes2)) return schemaMutationErrorResponse(regRes2.error); try { await uploadSchemaRegistry(stream, regRes2.value); } catch { return json(500, { error: { code: "internal", message: "schema upload failed" } }); } return json(200, regRes2.value); } if (update.schema === undefined && update.interpreter !== undefined && update.routingKey === undefined) { const regRes2 = registry.updateInterpreterResult(stream, update.interpreter ?? null); if (Result17.isError(regRes2)) return schemaMutationErrorResponse(regRes2.error); try { await uploadSchemaRegistry(stream, regRes2.value); } catch { return json(500, { error: { code: "internal", message: "schema upload failed" } }); } return json(200, regRes2.value); } if (update.schema === undefined && update.routingKey !== undefined && update.interpreter !== undefined) { const routingRes = registry.updateRoutingKeyResult(stream, update.routingKey ?? null); if (Result17.isError(routingRes)) return schemaMutationErrorResponse(routingRes.error); const interpreterRes = registry.updateInterpreterResult(stream, update.interpreter ?? null); if (Result17.isError(interpreterRes)) return schemaMutationErrorResponse(interpreterRes.error); try { await uploadSchemaRegistry(stream, interpreterRes.value); } catch { return json(500, { error: { code: "internal", message: "schema upload failed" } }); } return json(200, interpreterRes.value); } const regRes = registry.updateRegistryResult(stream, srow, update); if (Result17.isError(regRes)) return schemaMutationErrorResponse(regRes.error); try { await uploadSchemaRegistry(stream, regRes.value); } catch { return json(500, { error: { code: "internal", message: "schema upload failed" } }); } return json(200, regRes.value); } return badRequest("unsupported method"); } if (touchMode) { const srow = db.getStream(stream); if (!srow || db.isDeleted(srow)) return notFound(); if (srow.expires_at_ms != null && db.nowMs() > srow.expires_at_ms) return notFound("stream expired"); const regRes = registry.getRegistryResult(stream); if (Result17.isError(regRes)) return schemaReadErrorResponse(regRes.error); const reg = regRes.value; if (!isTouchEnabled(reg.interpreter)) return notFound("touch not enabled"); const touchCfg = reg.interpreter.touch; if (touchMode.kind === "templates_activate") { if (req.method !== "POST") return badRequest("unsupported method"); let body; try { body = await req.json(); } catch { return badRequest("activate body must be valid JSON"); } const templatesRaw = body?.templates; if (!Array.isArray(templatesRaw) || templatesRaw.length === 0) { return badRequest("activate.templates must be a non-empty array"); } if (templatesRaw.length > 256) return badRequest("activate.templates too large (max 256)"); const ttlRaw = body?.inactivityTtlMs; const inactivityTtlMs = ttlRaw === undefined ? touchCfg.templates?.defaultInactivityTtlMs ?? 60 * 60 * 1000 : typeof ttlRaw === "number" && Number.isFinite(ttlRaw) && ttlRaw >= 0 ? Math.floor(ttlRaw) : null; if (inactivityTtlMs == null) return badRequest("activate.inactivityTtlMs must be a non-negative number (ms)"); const templates = []; for (const t of templatesRaw) { const entity = typeof t?.entity === "string" ? t.entity.trim() : ""; const fieldsRaw = t?.fields; if (entity === "" || !Array.isArray(fieldsRaw) || fieldsRaw.length === 0 || fieldsRaw.length > 3) continue; const fields = []; for (const f of fieldsRaw) { const name = typeof f?.name === "string" ? f.name.trim() : ""; const encoding = f?.encoding; if (name === "") continue; fields.push({ name, encoding }); } if (fields.length !== fieldsRaw.length) continue; templates.push({ entity, fields }); } if (templates.length !== templatesRaw.length) return badRequest("activate.templates contains invalid template definitions"); const limits = { maxActiveTemplatesPerStream: touchCfg.templates?.maxActiveTemplatesPerStream ?? 2048, maxActiveTemplatesPerEntity: touchCfg.templates?.maxActiveTemplatesPerEntity ?? 256 }; const activeFromTouchOffset = touch.getOrCreateJournal(stream, touchCfg).getCursor(); const res = touch.activateTemplates({ stream, touchCfg, baseStreamNextOffset: srow.next_offset, activeFromTouchOffset, templates, inactivityTtlMs }); return json(200, { activated: res.activated, denied: res.denied, limits }); } if (touchMode.kind === "meta") { if (req.method !== "GET") return badRequest("unsupported method"); let activeTemplates = 0; try { const row = db.db.query(`SELECT COUNT(*) as cnt FROM live_templates WHERE stream=? AND state='active';`).get(stream); activeTemplates = Number(row?.cnt ?? 0); } catch { activeTemplates = 0; } const meta = touch.getOrCreateJournal(stream, touchCfg).getMeta(); const runtime2 = touch.getTouchRuntimeSnapshot({ stream, touchCfg }); const interp = db.getStreamInterpreter(stream); return json(200, { ...meta, coarseIntervalMs: touchCfg.coarseIntervalMs ?? 100, touchCoalesceWindowMs: touchCfg.touchCoalesceWindowMs ?? 100, activeTemplates, lagSourceOffsets: runtime2.lagSourceOffsets, touchMode: runtime2.touchMode, walScannedThrough: interp ? encodeOffset(srow.epoch, interp.interpreted_through) : null, bucketMaxSourceOffsetSeq: meta.bucketMaxSourceOffsetSeq, hotFineKeys: runtime2.hotFineKeys, hotTemplates: runtime2.hotTemplates, hotFineKeysActive: runtime2.hotFineKeysActive, hotFineKeysGrace: runtime2.hotFineKeysGrace, hotTemplatesActive: runtime2.hotTemplatesActive, hotTemplatesGrace: runtime2.hotTemplatesGrace, fineWaitersActive: runtime2.fineWaitersActive, coarseWaitersActive: runtime2.coarseWaitersActive, broadFineWaitersActive: runtime2.broadFineWaitersActive, hotKeyFilteringEnabled: runtime2.hotKeyFilteringEnabled, hotTemplateFilteringEnabled: runtime2.hotTemplateFilteringEnabled, scanRowsTotal: runtime2.scanRowsTotal, scanBatchesTotal: runtime2.scanBatchesTotal, scannedButEmitted0BatchesTotal: runtime2.scannedButEmitted0BatchesTotal, interpretedThroughDeltaTotal: runtime2.interpretedThroughDeltaTotal, touchesEmittedTotal: runtime2.touchesEmittedTotal, touchesTableTotal: runtime2.touchesTableTotal, touchesTemplateTotal: runtime2.touchesTemplateTotal, fineTouchesDroppedDueToBudgetTotal: runtime2.fineTouchesDroppedDueToBudgetTotal, fineTouchesSkippedColdTemplateTotal: runtime2.fineTouchesSkippedColdTemplateTotal, fineTouchesSkippedColdKeyTotal: runtime2.fineTouchesSkippedColdKeyTotal, fineTouchesSkippedTemplateBucketTotal: runtime2.fineTouchesSkippedTemplateBucketTotal, waitTouchedTotal: runtime2.waitTouchedTotal, waitTimeoutTotal: runtime2.waitTimeoutTotal, waitStaleTotal: runtime2.waitStaleTotal, journalFlushesTotal: runtime2.journalFlushesTotal, journalNotifyWakeupsTotal: runtime2.journalNotifyWakeupsTotal, journalNotifyWakeMsTotal: runtime2.journalNotifyWakeMsTotal, journalNotifyWakeMsMax: runtime2.journalNotifyWakeMsMax, journalTimeoutsFiredTotal: runtime2.journalTimeoutsFiredTotal, journalTimeoutSweepMsTotal: runtime2.journalTimeoutSweepMsTotal }); } if (touchMode.kind === "wait") { if (req.method !== "POST") return badRequest("unsupported method"); const waitStartMs = Date.now(); let body; try { body = await req.json(); } catch { return badRequest("wait body must be valid JSON"); } const keysRaw = body?.keys; const cursorRaw = body?.cursor; const timeoutMsRaw = body?.timeoutMs; if (keysRaw !== undefined && (!Array.isArray(keysRaw) || !keysRaw.every((k) => typeof k === "string" && k.trim() !== ""))) { return badRequest("wait.keys must be a non-empty string array when provided"); } const keys = Array.isArray(keysRaw) ? Array.from(new Set(keysRaw.map((k) => k.trim()))) : []; if (keys.length > 1024) return badRequest("wait.keys too large (max 1024)"); const keyIdsRaw = body?.keyIds; const keyIds = Array.isArray(keyIdsRaw) && keyIdsRaw.length > 0 ? Array.from(new Set(keyIdsRaw.map((x) => Number(x)).filter((n) => Number.isFinite(n) && Number.isInteger(n) && n >= 0 && n <= 4294967295))).map((n) => n >>> 0) : []; if (Array.isArray(keyIdsRaw) && keyIds.length !== keyIdsRaw.length) { return badRequest("wait.keyIds must be a non-empty uint32 array when provided"); } if (keys.length === 0 && keyIds.length === 0) return badRequest("wait requires keys or keyIds"); if (keyIds.length > 1024) return badRequest("wait.keyIds too large (max 1024)"); if (typeof cursorRaw !== "string" || cursorRaw.trim() === "") return badRequest("wait.cursor must be a non-empty string"); const cursor = cursorRaw.trim(); const timeoutMs = timeoutMsRaw === undefined ? 30000 : typeof timeoutMsRaw === "number" && Number.isFinite(timeoutMsRaw) ? Math.max(0, Math.min(120000, timeoutMsRaw)) : null; if (timeoutMs == null) return badRequest("wait.timeoutMs must be a number (ms)"); const templateIdsUsedRaw = body?.templateIdsUsed; if (Array.isArray(templateIdsUsedRaw) && !templateIdsUsedRaw.every((x) => typeof x === "string" && x.trim() !== "")) { return badRequest("wait.templateIdsUsed must be a string array"); } const templateIdsUsed = Array.isArray(templateIdsUsedRaw) && templateIdsUsedRaw.length > 0 ? Array.from(new Set(templateIdsUsedRaw.map((s) => typeof s === "string" ? s.trim() : "").filter((s) => s !== ""))) : []; const interestModeRaw = body?.interestMode; if (interestModeRaw !== undefined && interestModeRaw !== "fine" && interestModeRaw !== "coarse") { return badRequest("wait.interestMode must be 'fine' or 'coarse'"); } const interestMode = interestModeRaw === "coarse" ? "coarse" : "fine"; if (interestMode === "fine" && templateIdsUsed.length > 0) { touch.heartbeatTemplates({ stream, touchCfg, templateIdsUsed }); } const declareTemplatesRaw = body?.declareTemplates; if (Array.isArray(declareTemplatesRaw) && declareTemplatesRaw.length > 0) { if (declareTemplatesRaw.length > 256) return badRequest("wait.declareTemplates too large (max 256)"); const ttlRaw = body?.inactivityTtlMs; const inactivityTtlMs = ttlRaw === undefined ? touchCfg.templates?.defaultInactivityTtlMs ?? 60 * 60 * 1000 : typeof ttlRaw === "number" && Number.isFinite(ttlRaw) && ttlRaw >= 0 ? Math.floor(ttlRaw) : null; if (inactivityTtlMs == null) return badRequest("wait.inactivityTtlMs must be a non-negative number (ms)"); const templates = []; for (const t of declareTemplatesRaw) { const entity = typeof t?.entity === "string" ? t.entity.trim() : ""; const fieldsRaw = t?.fields; if (entity === "" || !Array.isArray(fieldsRaw) || fieldsRaw.length === 0 || fieldsRaw.length > 3) continue; const fields = []; for (const f of fieldsRaw) { const name = typeof f?.name === "string" ? f.name.trim() : ""; const encoding = f?.encoding; if (name === "") continue; fields.push({ name, encoding }); } if (fields.length !== fieldsRaw.length) continue; templates.push({ entity, fields }); } if (templates.length !== declareTemplatesRaw.length) return badRequest("wait.declareTemplates contains invalid template definitions"); const activeFromTouchOffset = touch.getOrCreateJournal(stream, touchCfg).getCursor(); touch.activateTemplates({ stream, touchCfg, baseStreamNextOffset: srow.next_offset, activeFromTouchOffset, templates, inactivityTtlMs }); } const j = touch.getOrCreateJournal(stream, touchCfg); const runtime2 = touch.getTouchRuntimeSnapshot({ stream, touchCfg }); let rawFineKeyIds = keyIds; if (keyIds.length === 0) { const parsedKeyIds = []; for (const key of keys) { const keyIdRes = touchKeyIdFromRoutingKeyResult(key); if (Result17.isError(keyIdRes)) return internalError(); parsedKeyIds.push(keyIdRes.value); } rawFineKeyIds = parsedKeyIds; } const templateWaitKeyIds = templateIdsUsed.length > 0 ? Array.from(new Set(templateIdsUsed.map((templateId) => templateKeyIdFor(templateId) >>> 0))) : []; let waitKeyIds = rawFineKeyIds; let effectiveWaitKind = "fineKey"; if (interestMode === "coarse") { effectiveWaitKind = "tableKey"; } else if (runtime2.touchMode === "restricted" && templateIdsUsed.length > 0) { effectiveWaitKind = "templateKey"; } else if (runtime2.touchMode === "coarseOnly" && templateIdsUsed.length > 0) { effectiveWaitKind = "tableKey"; } if (effectiveWaitKind === "templateKey") { waitKeyIds = templateWaitKeyIds; } else if (effectiveWaitKind === "tableKey" && templateIdsUsed.length > 0) { const entities = touch.resolveTemplateEntitiesForWait({ stream, templateIdsUsed }); waitKeyIds = Array.from(new Set(entities.map((entity) => tableKeyIdFor(entity) >>> 0))); } if (interestMode === "fine" && effectiveWaitKind === "fineKey" && templateWaitKeyIds.length > 0) { const merged = new Set; for (const keyId of waitKeyIds) merged.add(keyId >>> 0); for (const keyId of templateWaitKeyIds) merged.add(keyId >>> 0); waitKeyIds = Array.from(merged); } if (waitKeyIds.length === 0) { waitKeyIds = rawFineKeyIds; effectiveWaitKind = "fineKey"; } const hotInterestKeyIds = interestMode === "fine" ? rawFineKeyIds : waitKeyIds; const releaseHotInterest = touch.beginHotWaitInterest({ stream, touchCfg, keyIds: hotInterestKeyIds, templateIdsUsed, interestMode }); try { let sinceGen; if (cursor === "now") { sinceGen = j.getGeneration(); } else { const parsed = parseTouchCursor(cursor); if (!parsed) return badRequest("wait.cursor must be in the form : or 'now'"); if (parsed.epoch !== j.getEpoch()) { const latencyMs2 = Date.now() - waitStartMs; touch.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "stale", latencyMs: latencyMs2 }); return json(200, { stale: true, cursor: j.getCursor(), epoch: j.getEpoch(), generation: j.getGeneration(), effectiveWaitKind, bucketMaxSourceOffsetSeq: j.getLastFlushedSourceOffsetSeq().toString(), flushAtMs: j.getLastFlushAtMs(), bucketStartMs: j.getLastBucketStartMs(), error: { code: "stale", message: "cursor epoch mismatch; rerun/re-subscribe and start from cursor" } }); } sinceGen = parsed.generation; } const nowGen = j.getGeneration(); if (sinceGen > nowGen) sinceGen = nowGen; if (j.maybeTouchedSinceAny(waitKeyIds, sinceGen)) { const latencyMs2 = Date.now() - waitStartMs; touch.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs: latencyMs2 }); return json(200, { touched: true, cursor: j.getCursor(), effectiveWaitKind, bucketMaxSourceOffsetSeq: j.getLastFlushedSourceOffsetSeq().toString(), flushAtMs: j.getLastFlushAtMs(), bucketStartMs: j.getLastBucketStartMs() }); } const deadline = Date.now() + timeoutMs; const remaining = deadline - Date.now(); if (remaining <= 0) { const latencyMs2 = Date.now() - waitStartMs; touch.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "timeout", latencyMs: latencyMs2 }); return json(200, { touched: false, cursor: j.getCursor(), effectiveWaitKind, bucketMaxSourceOffsetSeq: j.getLastFlushedSourceOffsetSeq().toString(), flushAtMs: j.getLastFlushAtMs(), bucketStartMs: j.getLastBucketStartMs() }); } const afterGen = j.getGeneration(); const hit = await j.waitForAny({ keys: waitKeyIds, afterGeneration: afterGen, timeoutMs: remaining, signal: req.signal }); if (req.signal.aborted) return new Response(null, { status: 204 }); if (hit == null) { const latencyMs2 = Date.now() - waitStartMs; touch.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "timeout", latencyMs: latencyMs2 }); return json(200, { touched: false, cursor: j.getCursor(), effectiveWaitKind, bucketMaxSourceOffsetSeq: j.getLastFlushedSourceOffsetSeq().toString(), flushAtMs: j.getLastFlushAtMs(), bucketStartMs: j.getLastBucketStartMs() }); } const latencyMs = Date.now() - waitStartMs; touch.recordWaitMetrics({ stream, touchCfg, keysCount: waitKeyIds.length, outcome: "touched", latencyMs }); return json(200, { touched: true, cursor: j.getCursor(), effectiveWaitKind, bucketMaxSourceOffsetSeq: hit.bucketMaxSourceOffsetSeq.toString(), flushAtMs: hit.flushAtMs, bucketStartMs: hit.bucketStartMs }); } finally { releaseHotInterest(); } } } if (req.method === "PUT") { const streamClosed = parseStreamClosedHeader(req.headers.get("stream-closed")); const ttlHeader = req.headers.get("stream-ttl"); const expiresHeader = req.headers.get("stream-expires-at"); if (ttlHeader && expiresHeader) return badRequest("only one of Stream-TTL or Stream-Expires-At is allowed"); let ttlSeconds = null; let expiresAtMs = null; if (ttlHeader) { const ttlRes = parseStreamTtlSeconds(ttlHeader); if (Result17.isError(ttlRes)) return badRequest(ttlRes.error.message); ttlSeconds = ttlRes.value; expiresAtMs = db.nowMs() + BigInt(ttlSeconds) * 1000n; } else if (expiresHeader) { const expiresRes = parseTimestampMsResult(expiresHeader); if (Result17.isError(expiresRes)) return badRequest(expiresRes.error.message); expiresAtMs = expiresRes.value; } const contentType = normalizeContentType(req.headers.get("content-type")) ?? "application/octet-stream"; const routingKeyHeader = req.headers.get("stream-key"); const memReject = rejectIfMemoryLimited(); if (memReject) return memReject; const ab = await req.arrayBuffer(); if (ab.byteLength > cfg.appendMaxBodyBytes) return tooLarge(`body too large (max ${cfg.appendMaxBodyBytes})`); const bodyBytes = new Uint8Array(ab); let srow = db.getStream(stream); if (srow && db.isDeleted(srow)) { db.hardDeleteStream(stream); srow = null; } if (srow && srow.expires_at_ms != null && db.nowMs() > srow.expires_at_ms) { db.hardDeleteStream(stream); srow = null; } if (srow) { const existingClosed = srow.closed !== 0; const existingContentType = normalizeContentType(srow.content_type) ?? srow.content_type; const ttlMatch = ttlSeconds != null ? srow.ttl_seconds != null && srow.ttl_seconds === ttlSeconds : expiresAtMs != null ? srow.ttl_seconds == null && srow.expires_at_ms != null && srow.expires_at_ms === expiresAtMs : srow.ttl_seconds == null && srow.expires_at_ms == null; if (existingContentType !== contentType || existingClosed !== streamClosed || !ttlMatch) { return conflict("stream config mismatch"); } const tailOffset2 = encodeOffset(srow.epoch, srow.next_offset - 1n); const headers2 = { "content-type": existingContentType, "stream-next-offset": tailOffset2 }; if (existingClosed) headers2["stream-closed"] = "true"; if (srow.expires_at_ms != null) headers2["stream-expires-at"] = new Date(Number(srow.expires_at_ms)).toISOString(); return new Response(null, { status: 200, headers: withNosniff(headers2) }); } db.ensureStream(stream, { contentType, expiresAtMs, ttlSeconds, closed: false }); let lastOffset = -1n; let appendedRows = 0; let closedNow = false; if (bodyBytes.byteLength > 0) { const rowsRes = buildAppendRowsResult(stream, bodyBytes, contentType, routingKeyHeader, true); if (Result17.isError(rowsRes)) { if (rowsRes.error.status === 500) return internalError(); return badRequest(rowsRes.error.message); } const rows = rowsRes.value.rows; appendedRows = rows.length; if (rows.length > 0 || streamClosed) { const appendRes = await enqueueAppend({ stream, baseAppendMs: db.nowMs(), rows, contentType, close: streamClosed }); if (Result17.isError(appendRes)) { if (appendRes.error.kind === "overloaded") return json(429, { error: { code: "overloaded", message: "ingest queue full" } }); return json(500, { error: { code: "internal", message: "append failed" } }); } lastOffset = appendRes.value.lastOffset; closedNow = appendRes.value.closed; } } else if (streamClosed) { const appendRes = await enqueueAppend({ stream, baseAppendMs: db.nowMs(), rows: [], contentType, close: true }); if (Result17.isError(appendRes)) { if (appendRes.error.kind === "overloaded") return json(429, { error: { code: "overloaded", message: "ingest queue full" } }); return json(500, { error: { code: "internal", message: "close failed" } }); } lastOffset = appendRes.value.lastOffset; closedNow = appendRes.value.closed; } recordAppendOutcome({ stream, lastOffset, appendedRows, metricsBytes: bodyBytes.byteLength, ingestedBytes: bodyBytes.byteLength, touched: bodyBytes.byteLength > 0 || streamClosed, closed: closedNow }); const createdRow = db.getStream(stream); const tailOffset = encodeOffset(createdRow.epoch, createdRow.next_offset - 1n); const headers = { "content-type": contentType, "stream-next-offset": appendedRows > 0 || streamClosed ? encodeOffset(createdRow.epoch, lastOffset) : tailOffset, location: req.url }; if (streamClosed || closedNow) headers["stream-closed"] = "true"; if (createdRow.expires_at_ms != null) headers["stream-expires-at"] = new Date(Number(createdRow.expires_at_ms)).toISOString(); return new Response(null, { status: 201, headers: withNosniff(headers) }); } if (req.method === "DELETE") { const deleted = db.deleteStream(stream); if (!deleted) return notFound(); await uploader.publishManifest(stream); return new Response(null, { status: 204, headers: withNosniff() }); } if (req.method === "HEAD") { const srow = db.getStream(stream); if (!srow || db.isDeleted(srow)) return notFound(); if (srow.expires_at_ms != null && db.nowMs() > srow.expires_at_ms) return notFound("stream expired"); const tailOffset = encodeOffset(srow.epoch, srow.next_offset - 1n); const headers = { "content-type": normalizeContentType(srow.content_type) ?? srow.content_type, "stream-next-offset": tailOffset, "stream-end-offset": tailOffset, "cache-control": "no-store" }; if (srow.closed !== 0) headers["stream-closed"] = "true"; if (srow.ttl_seconds != null && srow.expires_at_ms != null) { const remainingMs = Number(srow.expires_at_ms - db.nowMs()); const remaining = Math.max(0, Math.ceil(remainingMs / 1000)); headers["stream-ttl"] = String(remaining); } if (srow.expires_at_ms != null) headers["stream-expires-at"] = new Date(Number(srow.expires_at_ms)).toISOString(); return new Response(null, { status: 200, headers: withNosniff(headers) }); } if (req.method === "POST") { const srow = db.getStream(stream); if (!srow || db.isDeleted(srow)) return notFound(); if (srow.expires_at_ms != null && db.nowMs() > srow.expires_at_ms) return notFound("stream expired"); const streamClosed = parseStreamClosedHeader(req.headers.get("stream-closed")); const streamContentType = normalizeContentType(srow.content_type) ?? srow.content_type; const producerId = req.headers.get("producer-id"); const producerEpochHeader = req.headers.get("producer-epoch"); const producerSeqHeader = req.headers.get("producer-seq"); let producer = null; if (producerId != null || producerEpochHeader != null || producerSeqHeader != null) { if (!producerId || producerId.trim() === "") return badRequest("invalid Producer-Id"); if (!producerEpochHeader || !producerSeqHeader) return badRequest("missing producer headers"); const epoch = parseNonNegativeInt(producerEpochHeader); const seq = parseNonNegativeInt(producerSeqHeader); if (epoch == null || seq == null) return badRequest("invalid producer headers"); producer = { id: producerId, epoch, seq }; } let streamSeq = null; const streamSeqRes = parseStreamSeqHeader(req.headers.get("stream-seq")); if (Result17.isError(streamSeqRes)) return badRequest(streamSeqRes.error.message); streamSeq = streamSeqRes.value; const tsHdr = req.headers.get("stream-timestamp"); let baseAppendMs = db.nowMs(); if (tsHdr) { const tsRes = parseTimestampMsResult(tsHdr); if (Result17.isError(tsRes)) return badRequest(tsRes.error.message); baseAppendMs = tsRes.value; } const memReject = rejectIfMemoryLimited(); if (memReject) return memReject; const ab = await req.arrayBuffer(); if (ab.byteLength > cfg.appendMaxBodyBytes) return tooLarge(`body too large (max ${cfg.appendMaxBodyBytes})`); const bodyBytes = new Uint8Array(ab); const isCloseOnly = streamClosed && bodyBytes.byteLength === 0; if (bodyBytes.byteLength === 0 && !streamClosed) return badRequest("empty body"); let reqContentType = normalizeContentType(req.headers.get("content-type")); if (!isCloseOnly && !reqContentType) return badRequest("missing content-type"); const routingKeyHeader = req.headers.get("stream-key"); let rows = []; if (!isCloseOnly) { const rowsRes = buildAppendRowsResult(stream, bodyBytes, reqContentType, routingKeyHeader, false); if (Result17.isError(rowsRes)) { if (rowsRes.error.status === 500) return internalError(); return badRequest(rowsRes.error.message); } rows = rowsRes.value.rows; } const appendRes = await enqueueAppend({ stream, baseAppendMs, rows, contentType: reqContentType ?? streamContentType, streamSeq, producer, close: streamClosed }); if (Result17.isError(appendRes)) { const err = appendRes.error; if (err.kind === "overloaded") return json(429, { error: { code: "overloaded", message: "ingest queue full" } }); if (err.kind === "gone") return notFound("stream expired"); if (err.kind === "not_found") return notFound(); if (err.kind === "content_type_mismatch") return conflict("content-type mismatch"); if (err.kind === "stream_seq") { return conflict("sequence mismatch", { "stream-expected-seq": err.expected, "stream-received-seq": err.received }); } if (err.kind === "closed") { const headers2 = { "stream-next-offset": encodeOffset(srow.epoch, err.lastOffset), "stream-closed": "true" }; return new Response(null, { status: 409, headers: withNosniff(headers2) }); } if (err.kind === "producer_stale_epoch") { return new Response(null, { status: 403, headers: withNosniff({ "producer-epoch": String(err.producerEpoch) }) }); } if (err.kind === "producer_gap") { return new Response(null, { status: 409, headers: withNosniff({ "producer-expected-seq": String(err.expected), "producer-received-seq": String(err.received) }) }); } if (err.kind === "producer_epoch_seq") return badRequest("invalid producer sequence"); return json(500, { error: { code: "internal", message: "append failed" } }); } const res = appendRes.value; const appendBytes = rows.reduce((acc, r) => acc + r.payload.byteLength, 0); recordAppendOutcome({ stream, lastOffset: res.lastOffset, appendedRows: res.appendedRows, metricsBytes: appendBytes, ingestedBytes: bodyBytes.byteLength, touched: true, closed: res.closed }); const headers = { "stream-next-offset": encodeOffset(srow.epoch, res.lastOffset) }; if (res.closed) headers["stream-closed"] = "true"; if (producer && res.producer) { headers["producer-epoch"] = String(res.producer.epoch); headers["producer-seq"] = String(res.producer.seq); } const status = producer && res.appendedRows > 0 ? 200 : 204; return new Response(null, { status, headers: withNosniff(headers) }); } if (req.method === "GET") { const srow = db.getStream(stream); if (!srow || db.isDeleted(srow)) return notFound(); if (srow.expires_at_ms != null && db.nowMs() > srow.expires_at_ms) return notFound("stream expired"); const streamContentType = normalizeContentType(srow.content_type) ?? srow.content_type; const isJsonStream = streamContentType === "application/json"; const fmtParam = url.searchParams.get("format"); let format = isJsonStream ? "json" : "raw"; if (fmtParam) { if (fmtParam !== "raw" && fmtParam !== "json") return badRequest("invalid format"); format = fmtParam; } if (format === "json" && !isJsonStream) return badRequest("invalid format"); const pathKey = pathKeyParam ?? null; const key = pathKey ?? url.searchParams.get("key"); const liveParam = url.searchParams.get("live") ?? ""; const cursorParam = url.searchParams.get("cursor"); let mode; if (liveParam === "" || liveParam === "false" || liveParam === "0") mode = "catchup"; else if (liveParam === "long-poll" || liveParam === "true" || liveParam === "1") mode = "long-poll"; else if (liveParam === "sse") mode = "sse"; else return badRequest("invalid live mode"); const timeout = url.searchParams.get("timeout") ?? url.searchParams.get("timeout_ms"); let timeoutMs = null; if (timeout) { if (/^[0-9]+$/.test(timeout)) { timeoutMs = Number(timeout); } else { const timeoutRes = parseDurationMsResult(timeout); if (Result17.isError(timeoutRes)) return badRequest("invalid timeout"); timeoutMs = timeoutRes.value; } } const hasOffsetParam = url.searchParams.has("offset"); let offset = url.searchParams.get("offset"); if (hasOffsetParam && (!offset || offset.trim() === "")) return badRequest("missing offset"); const sinceParam = url.searchParams.get("since"); if (!offset && sinceParam) { const sinceRes = parseTimestampMsResult(sinceParam); if (Result17.isError(sinceRes)) return badRequest(sinceRes.error.message); const seekRes = await reader.seekOffsetByTimestampResult(stream, sinceRes.value, key ?? null); if (Result17.isError(seekRes)) return readerErrorResponse(seekRes.error); offset = seekRes.value; } if (!offset) { if (mode === "catchup") offset = "-1"; else return badRequest("missing offset"); } let parsedOffset = null; if (offset !== "now") { const offsetRes = parseOffsetResult(offset); if (Result17.isError(offsetRes)) return badRequest(offsetRes.error.message); parsedOffset = offsetRes.value; } const ifNoneMatch = req.headers.get("if-none-match"); const sendBatch = async (batch2, cacheControl2, includeEtag) => { const upToDate = batch2.nextOffsetSeq === batch2.endOffsetSeq; const closedAtTail = srow.closed !== 0 && upToDate; const etag = includeEtag ? `W/"slice:${canonicalizeOffset(offset)}:${batch2.nextOffset}:key=${key ?? ""}:fmt=${format}"` : null; const baseHeaders = { "stream-next-offset": batch2.nextOffset, "stream-end-offset": batch2.endOffset, "cross-origin-resource-policy": "cross-origin" }; if (upToDate) baseHeaders["stream-up-to-date"] = "true"; if (closedAtTail) baseHeaders["stream-closed"] = "true"; if (cacheControl2) baseHeaders["cache-control"] = cacheControl2; if (etag) baseHeaders["etag"] = etag; if (srow.expires_at_ms != null) baseHeaders["stream-expires-at"] = new Date(Number(srow.expires_at_ms)).toISOString(); if (etag && ifNoneMatch && ifNoneMatch === etag) { return new Response(null, { status: 304, headers: withNosniff(baseHeaders) }); } if (format === "json") { const decoded = decodeJsonRecords(stream, batch2.records); if (Result17.isError(decoded)) { if (decoded.error.status === 500) return internalError(); return badRequest(decoded.error.message); } const body = JSON.stringify(decoded.value.values); metrics.recordRead(body.length, decoded.value.values.length); const headers2 = { "content-type": "application/json", ...baseHeaders }; return new Response(body, { status: 200, headers: withNosniff(headers2) }); } const outBytes = concatPayloads(batch2.records.map((r) => r.payload)); metrics.recordRead(outBytes.byteLength, batch2.records.length); const headers = { "content-type": streamContentType, ...baseHeaders }; const outBody = new Uint8Array(outBytes.byteLength); outBody.set(outBytes); return new Response(outBody, { status: 200, headers: withNosniff(headers) }); }; if (mode === "sse") { const baseCursor = srow.closed !== 0 ? null : computeCursor(Date.now(), cursorParam); const dataEncoding = isTextContentType(streamContentType) ? "text" : "base64"; const startOffsetSeq = offset === "now" ? srow.next_offset - 1n : offsetToSeqOrNeg1(parsedOffset); const startOffset = offset === "now" ? encodeOffset(srow.epoch, startOffsetSeq) : canonicalizeOffset(offset); const encoder = new TextEncoder; let aborted = false; const abortController = new AbortController; const streamBody = new ReadableStream({ start(controller) { (async () => { const fail = (message) => { if (aborted) return; aborted = true; abortController.abort(); controller.error(new Error(message)); }; let currentOffset = startOffset; let currentSeq = startOffsetSeq; let first = true; while (!aborted) { let batch2; if (offset === "now" && first) { batch2 = { stream, format, key: key ?? null, requestOffset: startOffset, endOffset: startOffset, nextOffset: startOffset, endOffsetSeq: currentSeq, nextOffsetSeq: currentSeq, records: [] }; } else { const batchRes2 = await reader.readResult({ stream, offset: currentOffset, key: key ?? null, format }); if (Result17.isError(batchRes2)) { fail(batchRes2.error.message); return; } batch2 = batchRes2.value; } first = false; let ssePayload = ""; if (batch2.records.length > 0) { let dataPayload = ""; if (format === "json") { const decoded = decodeJsonRecords(stream, batch2.records); if (Result17.isError(decoded)) { fail(decoded.error.message); return; } dataPayload = JSON.stringify(decoded.value.values); } else { const outBytes = concatPayloads(batch2.records.map((r) => r.payload)); dataPayload = dataEncoding === "base64" ? Buffer.from(outBytes).toString("base64") : new TextDecoder().decode(outBytes); } ssePayload += encodeSseEvent("data", dataPayload); } const upToDate = batch2.nextOffsetSeq === batch2.endOffsetSeq; const latest = db.getStream(stream); const closedNow = !!latest && latest.closed !== 0 && upToDate; const control = { streamNextOffset: batch2.nextOffset }; if (upToDate) control.upToDate = true; if (closedNow) control.streamClosed = true; if (!closedNow && baseCursor) control.streamCursor = baseCursor; ssePayload += encodeSseEvent("control", JSON.stringify(control)); controller.enqueue(encoder.encode(ssePayload)); if (closedNow) break; currentOffset = batch2.nextOffset; currentSeq = batch2.nextOffsetSeq; if (!upToDate) continue; const sseWaitMs = timeoutMs == null ? 30000 : timeoutMs; await notifier.waitFor(stream, currentSeq, sseWaitMs, abortController.signal); } if (!aborted) controller.close(); })().catch((err) => { if (!aborted) controller.error(err); }); }, cancel() { aborted = true; abortController.abort(); } }); const headers = { "content-type": "text/event-stream", "cache-control": "no-cache", "cross-origin-resource-policy": "cross-origin", "stream-next-offset": startOffset, "stream-end-offset": encodeOffset(srow.epoch, srow.next_offset - 1n) }; if (dataEncoding === "base64") headers["stream-sse-data-encoding"] = "base64"; return new Response(streamBody, { status: 200, headers: withNosniff(headers) }); } const defaultLongPollTimeoutMs = 3000; if (offset === "now") { const tailOffset = encodeOffset(srow.epoch, srow.next_offset - 1n); if (srow.closed !== 0) { if (mode === "long-poll") { const headers3 = { "stream-next-offset": tailOffset, "stream-end-offset": tailOffset, "stream-up-to-date": "true", "stream-closed": "true", "cache-control": "no-store" }; if (srow.expires_at_ms != null) headers3["stream-expires-at"] = new Date(Number(srow.expires_at_ms)).toISOString(); return new Response(null, { status: 204, headers: withNosniff(headers3) }); } const headers2 = { "content-type": streamContentType, "stream-next-offset": tailOffset, "stream-end-offset": tailOffset, "stream-up-to-date": "true", "stream-closed": "true", "cache-control": "no-store", "cross-origin-resource-policy": "cross-origin" }; if (srow.expires_at_ms != null) headers2["stream-expires-at"] = new Date(Number(srow.expires_at_ms)).toISOString(); const body2 = format === "json" ? "[]" : ""; return new Response(body2, { status: 200, headers: withNosniff(headers2) }); } if (mode === "long-poll") { const deadline = Date.now() + (timeoutMs ?? defaultLongPollTimeoutMs); let currentOffset = tailOffset; while (true) { const batchRes2 = await reader.readResult({ stream, offset: currentOffset, key: key ?? null, format }); if (Result17.isError(batchRes2)) return readerErrorResponse(batchRes2.error); const batch2 = batchRes2.value; if (batch2.records.length > 0) { const cursor = computeCursor(Date.now(), cursorParam); const resp = await sendBatch(batch2, "no-store", false); const headers3 = new Headers(resp.headers); headers3.set("stream-cursor", cursor); return new Response(resp.body, { status: resp.status, headers: headers3 }); } const latest2 = db.getStream(stream); if (latest2 && latest2.closed !== 0 && batch2.nextOffsetSeq === batch2.endOffsetSeq) { const latestTail2 = encodeOffset(latest2.epoch, latest2.next_offset - 1n); const headers3 = { "stream-next-offset": latestTail2, "stream-end-offset": latestTail2, "stream-up-to-date": "true", "stream-closed": "true", "cache-control": "no-store" }; if (latest2.expires_at_ms != null) headers3["stream-expires-at"] = new Date(Number(latest2.expires_at_ms)).toISOString(); return new Response(null, { status: 204, headers: withNosniff(headers3) }); } const remaining = deadline - Date.now(); if (remaining <= 0) break; currentOffset = batch2.nextOffset; await notifier.waitFor(stream, batch2.endOffsetSeq, remaining, req.signal); if (req.signal.aborted) return new Response(null, { status: 204 }); } const latest = db.getStream(stream); const latestTail = latest ? encodeOffset(latest.epoch, latest.next_offset - 1n) : tailOffset; const headers2 = { "stream-next-offset": latestTail, "stream-end-offset": latestTail, "stream-up-to-date": "true", "cache-control": "no-store" }; if (latest && latest.closed !== 0) headers2["stream-closed"] = "true"; else headers2["stream-cursor"] = computeCursor(Date.now(), cursorParam); if (latest && latest.expires_at_ms != null) headers2["stream-expires-at"] = new Date(Number(latest.expires_at_ms)).toISOString(); return new Response(null, { status: 204, headers: withNosniff(headers2) }); } const headers = { "content-type": streamContentType, "stream-next-offset": tailOffset, "stream-end-offset": tailOffset, "stream-up-to-date": "true", "cache-control": "no-store", "cross-origin-resource-policy": "cross-origin" }; const body = format === "json" ? "[]" : ""; return new Response(body, { status: 200, headers: withNosniff(headers) }); } if (mode === "long-poll") { const deadline = Date.now() + (timeoutMs ?? defaultLongPollTimeoutMs); let currentOffset = offset; while (true) { const batchRes2 = await reader.readResult({ stream, offset: currentOffset, key: key ?? null, format }); if (Result17.isError(batchRes2)) return readerErrorResponse(batchRes2.error); const batch2 = batchRes2.value; if (batch2.records.length > 0) { const cursor = computeCursor(Date.now(), cursorParam); const resp = await sendBatch(batch2, "no-store", false); const headers2 = new Headers(resp.headers); headers2.set("stream-cursor", cursor); return new Response(resp.body, { status: resp.status, headers: headers2 }); } const latest2 = db.getStream(stream); if (latest2 && latest2.closed !== 0 && batch2.nextOffsetSeq === batch2.endOffsetSeq) { const latestTail2 = encodeOffset(latest2.epoch, latest2.next_offset - 1n); const headers2 = { "stream-next-offset": latestTail2, "stream-end-offset": latestTail2, "stream-up-to-date": "true", "stream-closed": "true", "cache-control": "no-store" }; if (latest2.expires_at_ms != null) headers2["stream-expires-at"] = new Date(Number(latest2.expires_at_ms)).toISOString(); return new Response(null, { status: 204, headers: withNosniff(headers2) }); } const remaining = deadline - Date.now(); if (remaining <= 0) break; currentOffset = batch2.nextOffset; await notifier.waitFor(stream, batch2.endOffsetSeq, remaining, req.signal); if (req.signal.aborted) return new Response(null, { status: 204 }); } const latest = db.getStream(stream); const latestTail = latest ? encodeOffset(latest.epoch, latest.next_offset - 1n) : currentOffset; const headers = { "stream-next-offset": latestTail, "stream-end-offset": latestTail, "stream-up-to-date": "true", "cache-control": "no-store" }; if (latest && latest.closed !== 0) headers["stream-closed"] = "true"; else headers["stream-cursor"] = computeCursor(Date.now(), cursorParam); if (latest && latest.expires_at_ms != null) headers["stream-expires-at"] = new Date(Number(latest.expires_at_ms)).toISOString(); return new Response(null, { status: 204, headers: withNosniff(headers) }); } const batchRes = await reader.readResult({ stream, offset, key: key ?? null, format }); if (Result17.isError(batchRes)) return readerErrorResponse(batchRes.error); const batch = batchRes.value; const cacheControl = "immutable, max-age=31536000"; return sendBatch(batch, cacheControl, true); } return badRequest("unsupported method"); } return notFound(); } catch (e) { const msg = String(e?.message ?? e); if (!closing && !msg.includes("Statement has finalized")) { console.error("request failed", e); } return internalError(); } }; const close = () => { closing = true; touch.stop(); uploader.stop(true); indexer?.stop(); segmenter.stop(true); metricsEmitter.stop(); expirySweeper.stop(); ingest.stop(); memory.stop(); db.close(); }; return { fetch: fetch2, close, deps: { config: cfg, db, os: store, ingest, notifier, reader, segmenter, uploader, indexer, metrics, registry, touch, stats, backpressure, memory } }; } // src/objectstore/null.ts function disabled(op, key) { throw dsError(`object store disabled in local mode (${op}${key ? `: ${key}` : ""})`); } class NullObjectStore { async put(key, _data, _opts) { return disabled("put", key); } async putFile(key, _path, _size, _opts) { return disabled("putFile", key); } async get(key, _opts) { return disabled("get", key); } async head(key) { return disabled("head", key); } async delete(key) { return disabled("delete", key); } async list(prefix) { return disabled("list", prefix); } } // src/reader.ts import { existsSync as existsSync3, openSync, readSync, closeSync } from "node:fs"; // src/segment/format.ts import { Result as Result18 } from "better-result"; import { zstdCompressSync, zstdDecompressSync } from "node:zlib"; // src/util/bloom256.ts var BITS = 2048n; var MASK64 = (1n << 64n) - 1n; function fnv1a64(data) { let h = 14695981039346656037n; for (const b of data) { h ^= BigInt(b); h = h * 1099511628211n & MASK64; } return h; } function mix64(x) { x = x + 0x9e3779b97f4a7c15n & MASK64; x = (x ^ x >> 30n) * 0xbf58476d1ce4e5b9n & MASK64; x = (x ^ x >> 27n) * 0x94d049bb133111ebn & MASK64; x = x ^ x >> 31n; return x & MASK64; } class Bloom256 { bits; constructor(bits) { if (bits && bits.byteLength !== 32) throw dsError("bloom must be 32 bytes"); this.bits = bits ? new Uint8Array(bits) : new Uint8Array(32); } toBytes() { return new Uint8Array(this.bits); } add(keyUtf8) { if (keyUtf8.byteLength === 0) return; const h1 = fnv1a64(keyUtf8); const h2 = mix64(h1 ^ 0xa0761d6478bd642fn); for (let i = 0;i < 3; i++) { const idx = Number((h1 + BigInt(i) * h2) % BITS); const byte = idx >> 3; const bit = idx & 7; this.bits[byte] |= 1 << bit; } } maybeHas(keyUtf8) { if (keyUtf8.byteLength === 0) return true; const h1 = fnv1a64(keyUtf8); const h2 = mix64(h1 ^ 0xa0761d6478bd642fn); for (let i = 0;i < 3; i++) { const idx = Number((h1 + BigInt(i) * h2) % BITS); const byte = idx >> 3; const bit = idx & 7; if ((this.bits[byte] & 1 << bit) === 0) return false; } return true; } } // src/util/crc32c.ts var POLY = 2197175160; var TABLE = null; function makeTable() { const t = new Uint32Array(256); for (let i = 0;i < 256; i++) { let c = i; for (let k = 0;k < 8; k++) { c = c & 1 ? POLY ^ c >>> 1 : c >>> 1; } t[i] = c >>> 0; } return t; } function crc32c(buf) { if (!TABLE) TABLE = makeTable(); let c = 4294967295; for (const b of buf) { c = TABLE[(c ^ b) & 255] ^ c >>> 8; } return (c ^ 4294967295) >>> 0; } // src/segment/format.ts function invalidSegment(message) { return Result18.err({ kind: "invalid_segment_format", message }); } var DSB3_HEADER_BYTES = 68; var FOOTER_MAGIC = "DSF1"; var FOOTER_ENTRY_BYTES = 40; var FOOTER_TRAILER_BYTES = 8; function decodeBlockResult(blockBytes) { if (blockBytes.byteLength < DSB3_HEADER_BYTES) return invalidSegment("block too small"); if (blockBytes[0] !== 68 || blockBytes[1] !== 83 || blockBytes[2] !== 66 || blockBytes[3] !== 51) { return invalidSegment("bad block magic"); } const uncompressedLen = readU32BE(blockBytes, 4); const compressedLen = readU32BE(blockBytes, 8); const recordCount = readU32BE(blockBytes, 12); const bloom = blockBytes.slice(16, 48); const firstAppendNs = readU64BE(blockBytes, 48); const lastAppendNs = readU64BE(blockBytes, 56); const expectedCrc = readU32BE(blockBytes, 64); const payload = blockBytes.slice(DSB3_HEADER_BYTES, DSB3_HEADER_BYTES + compressedLen); if (payload.byteLength !== compressedLen) return invalidSegment("truncated block"); const actualCrc = crc32c(payload); if (actualCrc !== expectedCrc) return invalidSegment("crc mismatch"); let uncompressed; try { uncompressed = new Uint8Array(zstdDecompressSync(payload)); } catch (e) { return invalidSegment(String(e?.message ?? e)); } if (uncompressed.byteLength !== uncompressedLen) { return invalidSegment(`bad uncompressed len: got=${uncompressed.byteLength} expected=${uncompressedLen}`); } const records = []; let off = 0; for (let i = 0;i < recordCount; i++) { if (off + 8 + 4 > uncompressed.byteLength) return invalidSegment("truncated record"); const appendNs = readU64BE(uncompressed, off); off += 8; const keyLen = readU32BE(uncompressed, off); off += 4; if (off + keyLen + 4 > uncompressed.byteLength) return invalidSegment("truncated key"); const routingKey = uncompressed.slice(off, off + keyLen); off += keyLen; const dataLen = readU32BE(uncompressed, off); off += 4; if (off + dataLen > uncompressed.byteLength) return invalidSegment("truncated payload"); const payload2 = uncompressed.slice(off, off + dataLen); off += dataLen; records.push({ appendNs, routingKey, payload: payload2 }); } return Result18.ok({ recordCount, firstAppendNs, lastAppendNs, bloom, records }); } function parseFooter(segmentBytes) { if (segmentBytes.byteLength < FOOTER_TRAILER_BYTES) return null; const tail = segmentBytes.slice(segmentBytes.byteLength - 4); const tailMagic = String.fromCharCode(tail[0], tail[1], tail[2], tail[3]); if (tailMagic !== FOOTER_MAGIC) return null; const footerLen = readU32BE(segmentBytes, segmentBytes.byteLength - 8); if (footerLen <= 0 || footerLen + FOOTER_TRAILER_BYTES > segmentBytes.byteLength) return null; const footerStart = segmentBytes.byteLength - FOOTER_TRAILER_BYTES - footerLen; if (footerStart < 0) return null; const footer = segmentBytes.slice(footerStart, footerStart + footerLen); const parsed = parseFooterBytes(footer); return { footer: parsed, footerStart }; } function* iterateBlocksResult(segmentBytes) { const parsed = parseFooter(segmentBytes); const limit = parsed ? parsed.footerStart : segmentBytes.byteLength; let off = 0; while (off < limit) { if (off + DSB3_HEADER_BYTES > limit) { yield invalidSegment("truncated segment (block header)"); return; } const header = segmentBytes.slice(off, off + DSB3_HEADER_BYTES); const compressedLen = readU32BE(header, 8); const totalLen = DSB3_HEADER_BYTES + compressedLen; if (off + totalLen > limit) { yield invalidSegment("truncated segment (block payload)"); return; } const blockBytes = segmentBytes.slice(off, off + totalLen); const decodedRes = decodeBlockResult(blockBytes); if (Result18.isError(decodedRes)) { yield decodedRes; return; } yield Result18.ok({ blockOffset: off, blockBytes, decoded: decodedRes.value }); off += totalLen; } } function parseFooterBytes(footer) { if (footer.byteLength < 12) return null; const magic = String.fromCharCode(footer[0], footer[1], footer[2], footer[3]); if (magic !== FOOTER_MAGIC) return null; const version = readU32BE(footer, 4); const blockCount = readU32BE(footer, 8); const expectedLen = 12 + blockCount * FOOTER_ENTRY_BYTES; if (footer.byteLength !== expectedLen) return null; const blocks = []; let off = 12; for (let i = 0;i < blockCount; i++) { const blockOffset = Number(readU64BE(footer, off)); off += 8; const firstOffset = readU64BE(footer, off); off += 8; const recordCount = readU32BE(footer, off); off += 4; const compressedLen = readU32BE(footer, off); off += 4; const firstAppendNs = readU64BE(footer, off); off += 8; const lastAppendNs = readU64BE(footer, off); off += 8; blocks.push({ blockOffset, firstOffset, recordCount, compressedLen, firstAppendNs, lastAppendNs }); } return { version, blocks }; } function parseBlockHeaderResult(header) { if (header.byteLength < DSB3_HEADER_BYTES) return invalidSegment("block header too small"); if (header[0] !== 68 || header[1] !== 83 || header[2] !== 66 || header[3] !== 51) { return invalidSegment("bad block magic"); } const uncompressedLen = readU32BE(header, 4); const compressedLen = readU32BE(header, 8); const recordCount = readU32BE(header, 12); const bloom = header.slice(16, 48); const firstAppendNs = readU64BE(header, 48); const lastAppendNs = readU64BE(header, 56); const crc32cVal = readU32BE(header, 64); return Result18.ok({ uncompressedLen, compressedLen, recordCount, bloom, firstAppendNs, lastAppendNs, crc32c: crc32cVal }); } // src/util/stream_paths.ts import { createHash as createHash2 } from "node:crypto"; function streamHash16Hex(stream) { const full = createHash2("sha256").update(stream).digest(); return full.subarray(0, 16).toString("hex"); } function pad16(n) { const s = String(n); return s.padStart(16, "0"); } function segmentObjectKey(streamHash, segmentIndex) { return `streams/${streamHash}/segments/${pad16(segmentIndex)}.bin`; } // src/util/retry.ts function sleep(ms) { return new Promise((resolve2) => setTimeout(resolve2, ms)); } function withTimeout(p, ms) { if (ms <= 0) return p; return Promise.race([ p, new Promise((_, reject) => setTimeout(() => reject(dsError("timeout")), ms)) ]); } async function retry(fn, opts) { let attempt = 0; let delay = opts.baseDelayMs; for (;; ) { try { return await withTimeout(fn(), opts.timeoutMs); } catch (e) { attempt++; if (attempt > opts.retries) throw e; const jitter = Math.random() * 0.2 + 0.9; await sleep(Math.min(opts.maxDelayMs, Math.floor(delay * jitter))); delay = Math.min(opts.maxDelayMs, delay * 2); } } } // src/reader.ts import { Result as Result19 } from "better-result"; function errorMessage(e) { return String(e?.message ?? e); } function utf8Bytes(s) { return new TextEncoder().encode(s); } function objectKeyForSegment(seg) { const streamHash = streamHash16Hex(seg.stream); return segmentObjectKey(streamHash, seg.segment_index); } function readRangeFromFile(path, start, end) { const len = end - start + 1; const fd = openSync(path, "r"); try { const buf = Buffer.alloc(len); const bytesRead = readSync(fd, buf, 0, len, start); if (bytesRead !== len) throw dsError("short read"); return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); } finally { closeSync(fd); } } async function readSegmentRange(os, seg, start, end, diskCache, retryOpts) { const local = seg.local_path; if (existsSync3(local)) return readRangeFromFile(local, start, end); const objectKey = objectKeyForSegment(seg); if (diskCache && diskCache.has(objectKey)) { diskCache.recordHit(); diskCache.touch(objectKey); return readRangeFromFile(diskCache.getPath(objectKey), start, end); } if (diskCache) diskCache.recordMiss(); const bytes = await retry(async () => { const res = await os.get(objectKey, { range: { start, end } }); if (!res) throw dsError(`object store missing segment: ${objectKey}`); return res; }, retryOpts ?? { retries: 0, baseDelayMs: 0, maxDelayMs: 0, timeoutMs: 0 }); if (diskCache && start === 0 && end === seg.size_bytes - 1) { diskCache.put(objectKey, bytes); } return bytes; } async function loadSegmentBytes(os, seg, diskCache, retryOpts) { return readSegmentRange(os, seg, 0, seg.size_bytes - 1, diskCache, retryOpts); } async function loadSegmentFooter(os, seg, diskCache, retryOpts, footerCache) { const cacheKey = seg.segment_id; if (footerCache) { const cached = footerCache.get(cacheKey); if (cached) return cached; } if (seg.size_bytes < 8) return null; const tail = await readSegmentRange(os, seg, seg.size_bytes - 8, seg.size_bytes - 1, diskCache, retryOpts); const magic = String.fromCharCode(tail[4], tail[5], tail[6], tail[7]); if (magic !== "DSF1") return null; const footerLen = readU32BE(tail, 0); const footerStart = seg.size_bytes - 8 - footerLen; if (footerStart < 0) return null; const footerBytes = await readSegmentRange(os, seg, footerStart, footerStart + footerLen - 1, diskCache, retryOpts); const footer = parseFooterBytes(footerBytes); const result = { footer, footerStart }; if (footerCache) footerCache.set(cacheKey, result); return result; } class StreamReader { config; db; os; diskCache; footerCache; index; constructor(config, db, os, diskCache, index) { this.config = config; this.db = db; this.os = os; this.diskCache = diskCache; this.index = index; if (config.segmentFooterCacheEntries > 0) { this.footerCache = new LruCache(config.segmentFooterCacheEntries); } } cacheStats() { return this.diskCache ? this.diskCache.stats() : null; } retryOpts() { return { retries: this.config.objectStoreRetries, baseDelayMs: this.config.objectStoreBaseDelayMs, maxDelayMs: this.config.objectStoreMaxDelayMs, timeoutMs: this.config.objectStoreTimeoutMs }; } async seekOffsetByTimestampResult(stream, sinceMs, key) { const srow = this.db.getStream(stream); if (!srow || this.db.isDeleted(srow)) return Result19.err({ kind: "not_found", message: "not_found" }); if (srow.expires_at_ms != null && this.db.nowMs() > srow.expires_at_ms) { return Result19.err({ kind: "gone", message: "stream expired" }); } try { const sinceNs = sinceMs * 1000000n; const keyBytes = key ? utf8Bytes(key) : null; const segments = this.db.listSegmentsForStream(stream); for (const seg of segments) { const segBytes = await loadSegmentBytes(this.os, seg, this.diskCache, this.retryOpts()); let curOffset = seg.start_offset; for (const blockRes of iterateBlocksResult(segBytes)) { if (Result19.isError(blockRes)) return Result19.err({ kind: "internal", message: blockRes.error.message }); const { decoded } = blockRes.value; if (decoded.lastAppendNs < sinceNs) { curOffset += BigInt(decoded.recordCount); continue; } for (const r of decoded.records) { if (keyBytes && !bytesEqual(r.routingKey, keyBytes)) { curOffset += 1n; continue; } if (r.appendNs >= sinceNs) { const prev = curOffset - 1n; return Result19.ok(encodeOffset(srow.epoch, prev)); } curOffset += 1n; } } } const start = srow.sealed_through + 1n; const end = srow.next_offset - 1n; if (start <= end) { for (const rec of this.db.iterWalRange(stream, start, end, keyBytes ?? undefined)) { const tsNs = BigInt(rec.ts_ms) * 1000000n; if (tsNs >= sinceNs) { const off = BigInt(rec.offset) - 1n; return Result19.ok(encodeOffset(srow.epoch, off)); } } } const endOffsetNum = srow.next_offset - 1n; return Result19.ok(encodeOffset(srow.epoch, endOffsetNum)); } catch (e) { return Result19.err({ kind: "internal", message: errorMessage(e) }); } } async seekOffsetByTimestamp(stream, sinceMs, key) { const res = await this.seekOffsetByTimestampResult(stream, sinceMs, key); if (Result19.isError(res)) throw dsError(res.error.message); return res.value; } async readResult(args) { const { stream, offset, key, format } = args; const srow = this.db.getStream(stream); if (!srow || this.db.isDeleted(srow)) return Result19.err({ kind: "not_found", message: "not_found" }); if (srow.expires_at_ms != null && this.db.nowMs() > srow.expires_at_ms) { return Result19.err({ kind: "gone", message: "stream expired" }); } const epoch = srow.epoch; try { let finalize = function() { const scannedThrough = seq - 1n; const nextOffset = encodeOffset(epoch, scannedThrough); return { stream, format, key, requestOffset: offset, endOffset, nextOffset, endOffsetSeq: endOffsetNum, nextOffsetSeq: scannedThrough, records: results }; }; const parsed = parseOffsetResult(offset); if (Result19.isError(parsed)) { return Result19.err({ kind: "invalid_offset", message: parsed.error.message }); } const startOffsetExclusive = offsetToSeqOrNeg1(parsed.value); const desiredOffset = startOffsetExclusive + 1n; const endOffsetNum = srow.next_offset - 1n; const endOffset = encodeOffset(srow.epoch, endOffsetNum); const results = []; let bytesOut = 0; if (desiredOffset > endOffsetNum) { return Result19.ok({ stream, format, key, requestOffset: offset, endOffset, nextOffset: encodeOffset(srow.epoch, startOffsetExclusive), endOffsetSeq: endOffsetNum, nextOffsetSeq: startOffsetExclusive, records: [] }); } let seq = desiredOffset; const keyBytes = key ? utf8Bytes(key) : null; const indexInfo = keyBytes && this.index ? await this.index.candidateSegments(stream, keyBytes) : null; const candidateSegments = indexInfo?.segments ?? null; const indexedThrough = indexInfo?.indexedThrough ?? 0; const scanSegmentBytes = async (segBytes, seg) => { let curOffset = seg.start_offset; for (const blockRes of iterateBlocksResult(segBytes)) { if (Result19.isError(blockRes)) return Result19.err({ kind: "internal", message: blockRes.error.message }); const { decoded } = blockRes.value; if (keyBytes) { const bloom = new Bloom256(decoded.bloom); if (!bloom.maybeHas(keyBytes)) { curOffset += BigInt(decoded.recordCount); continue; } } for (const r of decoded.records) { if (curOffset < seq) { curOffset += 1n; continue; } if (curOffset > endOffsetNum) break; if (keyBytes && !bytesEqual(r.routingKey, keyBytes)) { curOffset += 1n; continue; } results.push({ offset: curOffset, payload: r.payload }); bytesOut += r.payload.byteLength; curOffset += 1n; if (results.length >= this.config.readMaxRecords || bytesOut >= this.config.readMaxBytes) { seq = curOffset; return Result19.ok(undefined); } } } return Result19.ok(undefined); }; while (seq <= endOffsetNum && seq <= srow.sealed_through) { const seg = this.db.findSegmentForOffset(stream, seq); if (!seg) { break; } if (keyBytes && candidateSegments && seg.segment_index < indexedThrough && !candidateSegments.has(seg.segment_index)) { seq = seg.end_offset + 1n; continue; } const preferFull = !keyBytes && this.config.readMaxBytes >= seg.size_bytes; if (preferFull) { const segBytes = await loadSegmentBytes(this.os, seg, this.diskCache, this.retryOpts()); const scanRes = await scanSegmentBytes(segBytes, seg); if (Result19.isError(scanRes)) return scanRes; if (results.length >= this.config.readMaxRecords || bytesOut >= this.config.readMaxBytes) return Result19.ok(finalize()); } else { const footerInfo = await loadSegmentFooter(this.os, seg, this.diskCache, this.retryOpts(), this.footerCache); if (!footerInfo || !footerInfo.footer) { const segBytes = await loadSegmentBytes(this.os, seg, this.diskCache, this.retryOpts()); const scanRes = await scanSegmentBytes(segBytes, seg); if (Result19.isError(scanRes)) return scanRes; if (results.length >= this.config.readMaxRecords || bytesOut >= this.config.readMaxBytes) return Result19.ok(finalize()); } else { const footer = footerInfo.footer; for (const entry of footer.blocks) { const blockStart = entry.firstOffset; const blockEnd = entry.firstOffset + BigInt(entry.recordCount) - 1n; if (blockEnd < seq) continue; if (blockStart > endOffsetNum) break; if (keyBytes) { const headerBytes = await readSegmentRange(this.os, seg, entry.blockOffset, entry.blockOffset + DSB3_HEADER_BYTES - 1, this.diskCache, this.retryOpts()); const headerRes = parseBlockHeaderResult(headerBytes); if (Result19.isError(headerRes)) return Result19.err({ kind: "internal", message: headerRes.error.message }); const header = headerRes.value; const bloom = new Bloom256(header.bloom); if (!bloom.maybeHas(keyBytes)) continue; } const totalLen = DSB3_HEADER_BYTES + entry.compressedLen; const blockBytes = await readSegmentRange(this.os, seg, entry.blockOffset, entry.blockOffset + totalLen - 1, this.diskCache, this.retryOpts()); const decodedRes = decodeBlockResult(blockBytes); if (Result19.isError(decodedRes)) return Result19.err({ kind: "internal", message: decodedRes.error.message }); const decoded = decodedRes.value; let curOffset = entry.firstOffset; for (const r of decoded.records) { if (curOffset < seq) { curOffset += 1n; continue; } if (curOffset > endOffsetNum) break; if (keyBytes && !bytesEqual(r.routingKey, keyBytes)) { curOffset += 1n; continue; } results.push({ offset: curOffset, payload: r.payload }); bytesOut += r.payload.byteLength; curOffset += 1n; if (results.length >= this.config.readMaxRecords || bytesOut >= this.config.readMaxBytes) { seq = curOffset; return Result19.ok(finalize()); } } } } } seq = seg.end_offset + 1n; } if (seq <= endOffsetNum) { let hitLimit = false; for (const rec of this.db.iterWalRange(stream, seq, endOffsetNum, keyBytes ?? undefined)) { const s = BigInt(rec.offset); const payload = rec.payload; results.push({ offset: s, payload }); bytesOut += payload.byteLength; if (results.length >= this.config.readMaxRecords || bytesOut >= this.config.readMaxBytes) { hitLimit = true; seq = s + 1n; break; } } if (!hitLimit) { seq = endOffsetNum + 1n; } } return Result19.ok(finalize()); } catch (e) { return Result19.err({ kind: "internal", message: errorMessage(e) }); } } async read(args) { const res = await this.readResult(args); if (Result19.isError(res)) throw dsError(res.error.message); return res.value; } } function bytesEqual(a, b) { if (a.byteLength !== b.byteLength) return false; for (let i = 0;i < a.byteLength; i++) if (a[i] !== b[i]) return false; return true; } // src/app_local.ts class NoopUploader { start() {} stop(_hard) {} countSegmentsWaiting() { return 0; } setHooks(_hooks) {} async publishManifest(_stream) {} } var noopSegmenter = { start() {}, stop(_hard) {} }; function createLocalApp(cfg, os, opts = {}) { return createAppCore(cfg, { stats: opts.stats, createRuntime: ({ config, db }) => { const store = os ?? new NullObjectStore; const reader = new StreamReader(config, db, store); return { store, reader, segmenter: noopSegmenter, uploader: new NoopUploader, uploadSchemaRegistry: async () => {}, start: () => {} }; } }); } // src/config.ts var KNOWN_DS_ENVS = new Set([ "DS_ROOT", "DS_HOST", "DS_DB_PATH", "DS_SEGMENT_MAX_BYTES", "DS_BLOCK_MAX_BYTES", "DS_SEGMENT_TARGET_ROWS", "DS_SEGMENT_MAX_INTERVAL_MS", "DS_SEGMENT_CHECK_MS", "DS_SEGMENTER_WORKERS", "DS_UPLOAD_CHECK_MS", "DS_UPLOAD_CONCURRENCY", "DS_SEGMENT_CACHE_MAX_BYTES", "DS_SEGMENT_FOOTER_CACHE_ENTRIES", "DS_INDEX_RUN_CACHE_MAX_BYTES", "DS_INDEX_RUN_MEM_CACHE_BYTES", "DS_INDEX_L0_SPAN", "DS_INDEX_BUILD_CONCURRENCY", "DS_INDEX_CHECK_MS", "DS_INDEX_COMPACTION_FANOUT", "DS_INDEX_MAX_LEVEL", "DS_INDEX_COMPACT_CONCURRENCY", "DS_INDEX_RETIRE_GEN_WINDOW", "DS_INDEX_RETIRE_MIN_MS", "DS_READ_MAX_BYTES", "DS_READ_MAX_RECORDS", "DS_APPEND_MAX_BODY_BYTES", "DS_INGEST_FLUSH_MS", "DS_INGEST_MAX_BATCH_REQS", "DS_INGEST_MAX_BATCH_BYTES", "DS_INGEST_MAX_QUEUE_REQS", "DS_INGEST_MAX_QUEUE_BYTES", "DS_INGEST_BUSY_MS", "DS_LOCAL_BACKLOG_MAX_BYTES", "DS_MEMORY_LIMIT_BYTES", "DS_MEMORY_LIMIT_MB", "DS_SQLITE_CACHE_BYTES", "DS_SQLITE_CACHE_MB", "DS_OBJECTSTORE_TIMEOUT_MS", "DS_OBJECTSTORE_RETRIES", "DS_OBJECTSTORE_RETRY_BASE_MS", "DS_OBJECTSTORE_RETRY_MAX_MS", "DS_LOCAL_DATA_ROOT", "DS_EXPIRY_SWEEP_MS", "DS_EXPIRY_SWEEP_LIMIT", "DS_METRICS_FLUSH_MS", "DS_INTERPRETER_WORKERS", "DS_INTERPRETER_CHECK_MS", "DS_INTERPRETER_MAX_BATCH_ROWS", "DS_INTERPRETER_MAX_BATCH_BYTES", "DS_STATS_INTERVAL_MS", "DS_BACKPRESSURE_BUDGET_MS", "DS_MOCK_R2_MAX_INMEM_BYTES", "DS_MOCK_R2_MAX_INMEM_MB", "DS_MOCK_R2_SPILL_DIR", "DS_BENCH_URL", "DS_BENCH_DURATION_MS", "DS_BENCH_INTERVAL_MS", "DS_BENCH_PAYLOAD_BYTES", "DS_BENCH_CONCURRENCY", "DS_BENCH_REQUEST_TIMEOUT_MS", "DS_BENCH_DRAIN_TIMEOUT_MS", "DS_BENCH_PAUSE_BACKGROUND", "DS_BENCH_YIELD_EVERY", "DS_BENCH_DEBUG", "DS_BENCH_SCENARIOS", "DS_MEMORY_STRESS_LIMITS_MB", "DS_MEMORY_STRESS_STATS_MS", "DS_MEMORY_STRESS_PORT_BASE", "DS_RK_EVENTS_MAX", "DS_RK_EVENTS_STEP", "DS_RK_PAYLOAD_BYTES", "DS_RK_APPEND_BATCH", "DS_RK_KEYS", "DS_RK_HOT_KEYS", "DS_RK_HOT_PCT", "DS_RK_PAYLOAD_POOL", "DS_RK_READ_ENTRIES", "DS_RK_WARM_READS", "DS_RK_SEGMENT_BYTES", "DS_RK_BLOCK_BYTES", "DS_RK_SEED", "DS_RK_R2_GET_DELAY_MS" ]); var warnedUnknownEnv = false; function warnUnknownEnv() { if (warnedUnknownEnv) return; warnedUnknownEnv = true; const unknown = []; for (const key of Object.keys(process.env)) { if (!key.startsWith("DS_")) continue; if (KNOWN_DS_ENVS.has(key)) continue; unknown.push(key); } if (unknown.length > 0) { unknown.sort(); console.warn(`[config] unknown DS_* environment variables: ${unknown.join(", ")}`); } } function envNum(name, def) { const v = process.env[name]; if (!v) return def; const n = Number(v); if (!Number.isFinite(n)) throw dsError(`invalid ${name}: ${v}`); return n; } function envBytes(name) { const v = process.env[name]; if (!v) return null; const n = Number(v); if (!Number.isFinite(n)) throw dsError(`invalid ${name}: ${v}`); return Math.max(0, Math.floor(n)); } function clampBytes(value, min, max) { if (!Number.isFinite(value)) return min; if (value < min) return min; if (value > max) return max; return value; } function loadConfig() { warnUnknownEnv(); const rootDir = process.env.DS_ROOT ?? "./ds-data"; const host = process.env.DS_HOST?.trim() || "127.0.0.1"; const bytesOverride = envBytes("DS_MEMORY_LIMIT_BYTES"); const mbOverride = envBytes("DS_MEMORY_LIMIT_MB"); const memoryLimitBytes = bytesOverride ?? (mbOverride != null ? mbOverride * 1024 * 1024 : 0); const backlogOverride = envBytes("DS_LOCAL_BACKLOG_MAX_BYTES"); const sqliteCacheBytesOverride = envBytes("DS_SQLITE_CACHE_BYTES"); const sqliteCacheMbOverride = envBytes("DS_SQLITE_CACHE_MB"); const indexMemOverride = envBytes("DS_INDEX_RUN_MEM_CACHE_BYTES"); const indexDiskOverride = envBytes("DS_INDEX_RUN_CACHE_MAX_BYTES"); const localBacklogMaxBytes = backlogOverride ?? 10 * 1024 * 1024 * 1024; const sqliteCacheBytes = sqliteCacheBytesOverride ?? (sqliteCacheMbOverride != null ? sqliteCacheMbOverride * 1024 * 1024 : memoryLimitBytes > 0 ? Math.floor(memoryLimitBytes * 0.25) : 0); const tunedIndexMem = indexMemOverride ?? (memoryLimitBytes > 0 ? clampBytes(Math.floor(memoryLimitBytes * 0.05), 8 * 1024 * 1024, 128 * 1024 * 1024) : 64 * 1024 * 1024); return { host, rootDir, dbPath: process.env.DS_DB_PATH ?? `${rootDir}/wal.sqlite`, segmentMaxBytes: envNum("DS_SEGMENT_MAX_BYTES", 16 * 1024 * 1024), blockMaxBytes: envNum("DS_BLOCK_MAX_BYTES", 256 * 1024), segmentTargetRows: envNum("DS_SEGMENT_TARGET_ROWS", 50000), segmentMaxIntervalMs: envNum("DS_SEGMENT_MAX_INTERVAL_MS", 0), segmentCheckIntervalMs: envNum("DS_SEGMENT_CHECK_MS", 250), segmenterWorkers: envNum("DS_SEGMENTER_WORKERS", 0), uploadIntervalMs: envNum("DS_UPLOAD_CHECK_MS", 250), uploadConcurrency: envNum("DS_UPLOAD_CONCURRENCY", 4), segmentCacheMaxBytes: envNum("DS_SEGMENT_CACHE_MAX_BYTES", 256 * 1024 * 1024), segmentFooterCacheEntries: envNum("DS_SEGMENT_FOOTER_CACHE_ENTRIES", 2048), indexRunCacheMaxBytes: indexDiskOverride ?? 256 * 1024 * 1024, indexRunMemoryCacheBytes: tunedIndexMem, indexL0SpanSegments: envNum("DS_INDEX_L0_SPAN", 16), indexBuildConcurrency: envNum("DS_INDEX_BUILD_CONCURRENCY", 4), indexCheckIntervalMs: envNum("DS_INDEX_CHECK_MS", 1000), indexCompactionFanout: envNum("DS_INDEX_COMPACTION_FANOUT", 16), indexMaxLevel: envNum("DS_INDEX_MAX_LEVEL", 4), indexCompactionConcurrency: envNum("DS_INDEX_COMPACT_CONCURRENCY", 4), indexRetireGenWindow: envNum("DS_INDEX_RETIRE_GEN_WINDOW", 2), indexRetireMinMs: envNum("DS_INDEX_RETIRE_MIN_MS", 5 * 60 * 1000), readMaxBytes: envNum("DS_READ_MAX_BYTES", 1 * 1024 * 1024), readMaxRecords: envNum("DS_READ_MAX_RECORDS", 1000), appendMaxBodyBytes: envNum("DS_APPEND_MAX_BODY_BYTES", 10 * 1024 * 1024), ingestFlushIntervalMs: envNum("DS_INGEST_FLUSH_MS", 10), ingestMaxBatchRequests: envNum("DS_INGEST_MAX_BATCH_REQS", 200), ingestMaxBatchBytes: envNum("DS_INGEST_MAX_BATCH_BYTES", 8 * 1024 * 1024), ingestMaxQueueRequests: envNum("DS_INGEST_MAX_QUEUE_REQS", 50000), ingestMaxQueueBytes: envNum("DS_INGEST_MAX_QUEUE_BYTES", 64 * 1024 * 1024), ingestBusyTimeoutMs: envNum("DS_INGEST_BUSY_MS", 5000), localBacklogMaxBytes, memoryLimitBytes, sqliteCacheBytes, objectStoreTimeoutMs: envNum("DS_OBJECTSTORE_TIMEOUT_MS", 5000), objectStoreRetries: envNum("DS_OBJECTSTORE_RETRIES", 3), objectStoreBaseDelayMs: envNum("DS_OBJECTSTORE_RETRY_BASE_MS", 50), objectStoreMaxDelayMs: envNum("DS_OBJECTSTORE_RETRY_MAX_MS", 2000), expirySweepIntervalMs: envNum("DS_EXPIRY_SWEEP_MS", 60000), expirySweepBatchLimit: envNum("DS_EXPIRY_SWEEP_LIMIT", 100), metricsFlushIntervalMs: envNum("DS_METRICS_FLUSH_MS", 1e4), interpreterWorkers: envNum("DS_INTERPRETER_WORKERS", 1), interpreterCheckIntervalMs: envNum("DS_INTERPRETER_CHECK_MS", 250), interpreterMaxBatchRows: envNum("DS_INTERPRETER_MAX_BATCH_ROWS", 500), interpreterMaxBatchBytes: envNum("DS_INTERPRETER_MAX_BATCH_BYTES", 4 * 1024 * 1024), port: envNum("PORT", 8080) }; } // src/local/state.ts import { closeSync as closeSync2, existsSync as existsSync4, mkdirSync as mkdirSync2, openSync as openSync2, readFileSync, readdirSync as readdirSync2, writeFileSync } from "node:fs"; import { join as join3 } from "node:path"; import lockfile from "proper-lockfile"; // src/local/paths.ts import envPaths from "env-paths"; import { join as join2, resolve as resolve2 } from "node:path"; function normalizeServerName(name) { const trimmed = (name ?? "default").trim(); if (trimmed.length === 0) return "default"; if (!/^[A-Za-z0-9._-]+$/.test(trimmed)) { throw dsError(`invalid server name: ${name}`); } return trimmed; } function getDurableStreamsDataRoot() { const override = process.env.DS_LOCAL_DATA_ROOT; if (override && override.trim().length > 0) return resolve2(override); const root = envPaths("prisma-dev").data; return join2(root, "durable-streams"); } function getServerDataDir(name) { return join2(getDurableStreamsDataRoot(), normalizeServerName(name)); } // src/local/state.ts function getDataDir(name) { return getServerDataDir(name); } function getDbPath(name) { return join3(getDataDir(name), "durable-streams.sqlite"); } function getServerDumpPath(name) { return join3(getDataDir(name), "server.json"); } function getLockPath(name) { return join3(getDataDir(name), "server.lock"); } function ensureFile(path) { const fd = openSync2(path, "a"); closeSync2(fd); } async function acquireLock(name) { const normalized = normalizeServerName(name); const dataDir = getDataDir(normalized); mkdirSync2(dataDir, { recursive: true }); const lockPath = getLockPath(normalized); ensureFile(lockPath); const release = await lockfile.lock(lockPath, { realpath: false, stale: 30000, retries: { retries: 0 } }); return async () => { try { await release(); } catch {} }; } function writeServerDump(name, dump) { const normalized = normalizeServerName(name); const dataDir = getDataDir(normalized); mkdirSync2(dataDir, { recursive: true }); writeFileSync(getServerDumpPath(normalized), `${JSON.stringify(dump, null, 2)} `, "utf8"); } // src/local/config.ts function buildLocalConfig(args) { const name = normalizeServerName(args.name); const dataDir = getDataDir(name); const dbPath = getDbPath(name); const base = loadConfig(); const port = args.port == null ? 0 : Math.max(0, Math.floor(args.port)); return { ...base, rootDir: dataDir, dbPath, port, segmentCacheMaxBytes: 0, segmenterWorkers: 0, interpreterCheckIntervalMs: Math.min(base.interpreterCheckIntervalMs, 5) }; } // src/local/http.ts import { createServer } from "node:http"; import { Readable } from "node:stream"; function hasBody(method) { const upper = method.toUpperCase(); return upper !== "GET" && upper !== "HEAD"; } function requestFromNode(req, opts) { const method = (req.method ?? "GET").toUpperCase(); const host = req.headers.host ?? `${opts.hostname ?? "127.0.0.1"}:${opts.port}`; const url = `http://${host}${req.url ?? "/"}`; const init = { method, headers: req.headers }; if (hasBody(method)) { init.body = Readable.toWeb(req); init.duplex = "half"; } return new Request(url, init); } async function writeNodeResponse(req, res, response) { res.statusCode = response.status; response.headers.forEach((value, key) => { res.setHeader(key, value); }); if (!response.body || req.method?.toUpperCase() === "HEAD") { res.end(); return; } const body = Readable.fromWeb(response.body); await new Promise((resolve3, reject) => { body.on("error", reject); res.on("error", reject); res.on("finish", () => resolve3()); body.pipe(res); }); } async function serveFetchHandler(fetchHandler, opts) { if (typeof globalThis.Bun !== "undefined") { const server2 = Bun.serve({ hostname: opts.hostname, port: opts.port, idleTimeout: 65, fetch: fetchHandler }); return { port: server2.port ?? opts.port, close: async () => { server2.stop(true); } }; } const hostname = opts.hostname ?? "127.0.0.1"; const server = createServer(async (req, res) => { try { const request = requestFromNode(req, { hostname, port: opts.port }); const response = await fetchHandler(request); await writeNodeResponse(req, res, response); } catch (err) { console.error("local fetch handler failed", err); res.statusCode = 500; res.setHeader("content-type", "application/json; charset=utf-8"); res.end(JSON.stringify({ error: { code: "internal", message: "internal server error" } })); } }); await new Promise((resolve3, reject) => { server.once("error", reject); server.listen(opts.port, hostname, () => { server.off("error", reject); resolve3(); }); }); const address = server.address(); const port = typeof address === "object" && address ? address.port : opts.port; return { port, close: async () => { await new Promise((resolve3, reject) => { server.close((err) => err ? reject(err) : resolve3()); }); } }; } // src/local/server.ts async function startLocalDurableStreamsServer(opts = {}) { const name = normalizeServerName(opts.name); const hostname = opts.hostname ?? "127.0.0.1"; const releaseLock = await acquireLock(name); let app = null; let http = null; let closed = false; try { const cfg = buildLocalConfig({ name, port: opts.port }); app = createLocalApp(cfg); http = await serveFetchHandler(app.fetch, { hostname, port: cfg.port }); const exportsPayload = { name, pid: process.pid, http: { port: http.port, url: `http://${hostname}:${http.port}` }, sqlite: { path: getDbPath(name) } }; const dump = { version: 1, name, pid: process.pid, startedAt: new Date().toISOString(), http: exportsPayload.http, sqlite: exportsPayload.sqlite }; writeServerDump(name, dump); return { exports: exportsPayload, close: async () => { if (closed) return; closed = true; try { if (http) await http.close(); } finally { try { app?.close(); } finally { await releaseLock(); } } } }; } catch (err) { try { if (http) await http.close(); } catch {} try { app?.close(); } catch {} await releaseLock(); throw err; } } export { parseLocalProcessOptions, startLocalDurableStreamsServer };