From 9db968b7d4ebdfe14af905de8716502c079fdc39 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Sun, 2 Aug 2026 23:45:12 +0200 Subject: [PATCH 1/2] chore(shared-dataflow): remove package (relocated to webrun-files) @statewalker/shared-dataflow moved (history-preserving) into the webrun-files monorepo as @statewalker/webrun-dataflow. Drop the package here and regenerate the submodule lockfile so its stale importer entry is gone. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/shared-dataflow/README.md | 551 -------------- packages/shared-dataflow/docs/use-cases.md | 20 - packages/shared-dataflow/package.json | 44 -- .../shared-dataflow/src/dataflow-graph.ts | 217 ------ .../src/in-memory-transaction-store.ts | 35 - .../src/in-memory-updates-store.ts | 233 ------ packages/shared-dataflow/src/index.ts | 23 - .../shared-dataflow/src/read-cell-updates.ts | 123 --- .../shared-dataflow/src/transaction-store.ts | 48 -- packages/shared-dataflow/src/types.ts | 8 - .../shared-dataflow/src/updates-manager.ts | 151 ---- packages/shared-dataflow/src/updates-store.ts | 156 ---- .../tests/dataflow-graph.test.ts | 260 ------- .../tests/in-memory-transaction-store.test.ts | 133 ---- .../tests/in-memory-updates-store.test.ts | 629 ---------------- .../tests/read-cell-updates.test.ts | 399 ---------- .../tests/updates-manager.test.ts | 703 ------------------ .../tests/updates-store-integration.test.ts | 462 ------------ packages/shared-dataflow/tsconfig.json | 28 - pnpm-lock.yaml | 15 - 20 files changed, 4238 deletions(-) delete mode 100644 packages/shared-dataflow/README.md delete mode 100644 packages/shared-dataflow/docs/use-cases.md delete mode 100644 packages/shared-dataflow/package.json delete mode 100644 packages/shared-dataflow/src/dataflow-graph.ts delete mode 100644 packages/shared-dataflow/src/in-memory-transaction-store.ts delete mode 100644 packages/shared-dataflow/src/in-memory-updates-store.ts delete mode 100644 packages/shared-dataflow/src/index.ts delete mode 100644 packages/shared-dataflow/src/read-cell-updates.ts delete mode 100644 packages/shared-dataflow/src/transaction-store.ts delete mode 100644 packages/shared-dataflow/src/types.ts delete mode 100644 packages/shared-dataflow/src/updates-manager.ts delete mode 100644 packages/shared-dataflow/src/updates-store.ts delete mode 100644 packages/shared-dataflow/tests/dataflow-graph.test.ts delete mode 100644 packages/shared-dataflow/tests/in-memory-transaction-store.test.ts delete mode 100644 packages/shared-dataflow/tests/in-memory-updates-store.test.ts delete mode 100644 packages/shared-dataflow/tests/read-cell-updates.test.ts delete mode 100644 packages/shared-dataflow/tests/updates-manager.test.ts delete mode 100644 packages/shared-dataflow/tests/updates-store-integration.test.ts delete mode 100644 packages/shared-dataflow/tsconfig.json diff --git a/packages/shared-dataflow/README.md b/packages/shared-dataflow/README.md deleted file mode 100644 index 44cba9e..0000000 --- a/packages/shared-dataflow/README.md +++ /dev/null @@ -1,551 +0,0 @@ -# @statewalker/shared-dataflow - -Signal-driven dataflow graph: forward impact propagation + filtered Kahn topological sort. Zero runtime dependencies. - -## The problem it solves - -Many real systems share the same shape: **something upstream changes, and a cascade of downstream work has to catch up** — in the right order, without redoing work that's already done, without losing progress if something fails, and without melting the machine when a thousand things change at once. - -Hand-rolling that for every pipeline ends up reinventing the same wheel: queues, watermarks, retry logic, ordering hacks, "is this already up to date?" checks, ad-hoc resumption flags. `shared-dataflow` is that wheel, factored out and made declarative. - -Concretely, it gives you: - -- **Incremental work by URI.** Each handler sees only what changed since *its* last successful run — not the whole world. Sweep a million-file repo and a no-op activation costs almost nothing. -- **Batching without saturation.** Handlers can choose to process a slice of the pending entries, return `false` ("more to do"), and be re-invoked on the next sweep. Backpressure is just "do less per call." -- **Resumable on failure.** A handler that throws or returns `false` leaves its bookmark untouched. Next activation replays exactly the same `updateId`, so progress picks up where it stopped — no compensating actions, no "did I already do this?" guesswork. -- **Guaranteed convergence.** As long as inputs eventually stop changing, the cascade reaches a fixed point where every cell's bookmark equals the latest upstream stamp. Re-runs over a quiet system are structural no-ops. -- **Order without coordination.** The topological sort guarantees a consumer only fires after every upstream producer that *also* has work to do has finished. No timestamps, no priorities, no race windows. -- **Deletes are just another signal.** Tombstone signals flow through the same cascade — no parallel "deletion pipeline" to maintain. -- **Asynchronous, decoupled stages.** The store mediates between handlers; nobody passes data hand-to-hand. Add a new consumer of an existing signal and the graph picks it up; remove one and nothing else cares. - -In one line: **describe the graph once, write idempotent handlers, and let the runtime turn "something changed" into the minimum correct cascade — every time, even after crashes.** - -See [docs/use-cases.md](docs/use-cases.md) for example domains beyond content pipelines — ETL, CI/CD, cache invalidation, IoT, ML, and more. - -## What it is - -A small TypeScript library that models a graph of *cells* connected by *signals*: - -- A **cell** declares the signals it reads (`inputs`) and the signals it produces (`outputs`). -- A **signal** can be produced by multiple cells and consumed by multiple cells. -- Given a set of changed signals, `getExecutionOrder` returns the impacted cells in a valid execution order. - -## Why it exists - -Captures a specific, opinionated execution semantics — **barrier synchronization, not "latest wins"**: - -> A consumer must run after **all** producers of its inputs that are themselves scheduled in this execution. - -This avoids races without requiring priorities or timestamps; ordering is purely structural. - -## How to use - -```ts -import { DataflowGraph } from "@statewalker/shared-dataflow"; - -const graph = new DataflowGraph([ - { id: "A", inputs: [], outputs: ["x", "n"] }, - { id: "B", inputs: ["n"], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: [] }, -]); - -graph.getExecutionOrder(["n"]); -// → ["B", "C"] (A produces n but is not impacted by changing n itself) - -graph.getExecutionOrder(["x"]); -// → ["C"] -``` - -## Examples - -### Diamond - -```ts -// A -// / \ -// B C -// \ / -// D -const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["x"], outputs: ["z"] }, - { id: "D", inputs: ["y", "z"], outputs: [] }, -]); - -g.getExecutionOrder(["s"]); -// → A first, then B and C in either order, then D -``` - -### Multi-producer barrier - -```ts -const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["s"], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: [] }, -]); - -g.getExecutionOrder(["s"]); -// → C runs after BOTH A and B (their order between themselves is free) -``` - -## Internals - -The algorithm runs in three phases on every call to `getExecutionOrder`: - -1. **Seed lookup** — for each changed signal, collect its direct consumers via the precomputed `signal → consumers` index. `O(|changed| + |seeds|)`. -2. **Forward propagation** — BFS through `cell.outputs → consumers` to grow the impacted set. Walks downstream only; producers of unchanged signals are not pulled in. `O(V_impacted + E_impacted)`. -3. **Filtered Kahn topological sort** — restrict the dependency graph to the impacted set: a cell depends on impacted producers of its inputs. Run Kahn's algorithm. Cycles confined to the impacted subgraph throw; cycles outside it are silently ignored. `O(V_impacted + E_impacted)`. - -### Precomputed indexes - -The constructor builds two `Map>` tables — `signalToConsumers` and `signalToProducers` — and never mutates them after construction. Per-execution work scales with the impacted subgraph, not the whole graph. - -### Why filter-at-runtime instead of precomputing transitive closure? - -Reachability (who is affected) can be precomputed, but **scheduling order** depends on which cells are *also* in the impacted set on this run — different changed-signal sets pull in different producer subsets. Reusing a static transitive closure would still require the per-execution dependency filter, so the savings are marginal for typical graphs and not worth the storage. - -### Constraints - -- All cell ids must be unique (constructor throws on duplicates). -- Self-loops (a cell whose output feeds its own input) are tolerated — the cell does not depend on itself. -- The impacted subgraph must be acyclic; otherwise `getExecutionOrder` throws. - -### Dependencies - -Zero runtime dependencies. Dev-only: `tsdown`, `vitest`, `typescript`, `rimraf`. - -## Transaction store - -Alongside the topology, this package also ships a small bookkeeping interface used by an updates manager that drives handler execution over the graph. - -### `TransactionStore` interface - -```ts -interface TransactionStore { - newTransactionId(): Promise; - setCellTransaction(cell: CellId, transactionId: number): Promise; - getCellTransaction(cell: CellId): Promise; - getCellsTransactions( - sinceTransactionId?: number, - ): AsyncGenerator<[cell: CellId, transactionId: number]>; - removeCellTransactions(cell: CellId): Promise; -} -``` - -- `newTransactionId` returns strictly increasing numbers across the lifetime of the store. -- `setCellTransaction` is called only after a handler returns `true` — failed/partial runs leave the cell's recorded transaction unchanged. -- `getCellTransaction` returns `0` for cells that have never been recorded. -- `getCellsTransactions(since)` yields cells with `recordedTx > since`; with no argument it yields all recorded cells. -- `removeCellTransactions` forgets a cell entirely (e.g., after a config change). - -### `InMemoryTransactionStore` - -Reference implementation backed by a single counter and a `Map`. State lives in this process; nothing persists across restarts. Suitable for tests and single-process use. - -```ts -import { InMemoryTransactionStore } from "@statewalker/shared-dataflow"; - -const store = new InMemoryTransactionStore(); -const tx = await store.newTransactionId(); // 1, 2, 3, ... -await store.setCellTransaction("ExtractContent", tx); -await store.getCellTransaction("ExtractContent"); // → tx -``` - -Persistent backends (SQL, KV) ship as separate packages and implement the same interface. - -## Updates store - -The third leaf of the package. `UpdatesStore` holds **two relations**, both keyed by signal channel: - -- `updates(signal, uri) → stamp` — "the URI changed on this signal at stamp `s`". Written by `setUpdate`, read by `readEntries` and `readUpdates`. -- `handled(signal, cell, uri) → stamp` — "this cell has caught up to that change as of stamp `s`". Written by `handleUpdate`, consulted (never yielded) by `readUpdates`, reset by `clearHandled`. - -Together with `DataflowGraph` (topology) and `TransactionStore` (per-cell last-success tx), it answers the questions handlers need: *"what changed on signal X that I haven't handled yet?"* and *"what am I marking changed on signal Y right now?"* — with each consumer tracking its progress **independently**, so two cells can consume the same `(signal, uri)` change without interfering. - -### `UpdatesStore` interface - -```ts -interface UpdateEntry { - signal: Signal; - uri: string; - stamp: number; -} - -interface HandledEntry { - signal: Signal; - uri: string; - cell: string; - stamp: number; -} - -type ReadOrderBy = "stamp" | "uri"; // default "stamp" - -interface UpdatesStore { - // --- reads --- - readEntries(opts: { - signal: Signal; - since: number; // exclusive: yields stamp > since - uriPrefix?: string; - orderBy?: ReadOrderBy; - }): AsyncIterable; - - readUpdates(opts: { - signal: Signal; - cell: string; // per-cell watermark; absent handled stamp == 0 - uriPrefix?: string; - orderBy?: ReadOrderBy; - }): AsyncIterable; - - // --- updates relation --- - setUpdate(entry: UpdateEntry): Promise; - setUpdates(entries: ReadonlyArray): Promise; - - // --- handled relation --- - handleUpdate(entry: HandledEntry): Promise; - handleUpdates(entries: ReadonlyArray): Promise; - clearHandled(key: { signal: Signal; cell: string }): Promise; - - // --- removal (cascades into handled) --- - removeUpdate(key: { signal: Signal; uri: string }): Promise; - removeUpdates(keys: ReadonlyArray<{ signal: Signal; uri: string }>): Promise; -} -``` - -Exact semantics: - -- **`setUpdate` — upsert by `(signal, uri)`, blind replace.** Each call overwrites the previous stamp for that pair; no history is retained. The store does **not** enforce monotonicity — a smaller stamp replaces a larger one. A non-finite stamp (`NaN`, `Infinity`) is rejected (throws). Entries are pure pointers `{ signal, uri, stamp }`; the data the URI addresses lives in the caller's domain store. -- **`handleUpdate` — upsert by `(signal, cell, uri)`, blind replace.** Records that `cell` has handled `(signal, uri)` at `stamp` (intended to be the upstream stamp the cell just observed). Same finite-stamp guard. **Never touches `updates` rows** and never affects another cell's handled rows. -- **`readEntries({ signal, since })` — raw, watermark-free read.** Yields every `updates` row on `signal` with `stamp > since` (strict). `since = 0` reads everything. No `cell` dimension. -- **`readUpdates({ signal, cell })` — per-cell diff.** For each `uri` present in `updates[signal]`, yields the entry iff its update stamp is strictly greater than that cell's handled stamp for the same uri (absent handled stamp treated as `0`). A uri present only as handled state (no `updates` row) is never yielded. -- **`clearHandled({ signal, cell })` — watermark reset.** Removes every handled row recorded by `cell` against `signal` (so all of that signal's updates re-appear in the cell's next `readUpdates`). Returns the number of rows removed. Touches no `updates` row and no other cell's handled rows — so resetting one consumer leaves siblings sharing the same input untouched. -- **`removeUpdate({ signal, uri })` — cascading delete.** Removes the `updates` row **and** every cell's handled row for that same `(signal, uri)`. Idempotent (no-op if absent). The cascade prevents a re-created uri from being masked by a stale handled stamp. -- **Ordering (`orderBy`).** Both reads default to `"stamp"` (update-stamp ascending). `"uri"` yields URI-ascending — the order `readCellUpdates` relies on to merge several per-signal streams with O(1) buffering. Same set either way; only the order differs. -- **URI-prefix filter.** `uriPrefix` (on both reads) restricts to entries whose `uri.startsWith(uriPrefix)`; an empty/absent prefix means no filter. Useful for "all files under folder X" or "all chunks of file Y" (when chunk URIs are `#`). -- **Batch ops.** `setUpdates` / `handleUpdates` / `removeUpdates` are exactly N sequential single calls in iteration order — no atomicity promise. - -### Two consumption models - -A cell needs a *watermark* — "how far have I caught up?" — to read only what's new. The store supports two, and you pick per cell: - -**1. Coarse, per-cell transaction watermark** (`readEntries` + `since: updateId`). The watermark is the cell's last successful `transactionId` (from `TransactionStore`, supplied to the handler as `updateId`). Simple and adequate when a cell consumes a single signal and "everything stamped after my last successful run" is the right delta. - -```ts -function newExtractor(deps: { files: FilesApi; updatesStore: UpdatesStore }): CellHandler { - return async ({ updateId, transactionId }) => { - for await (const { uri } of deps.updatesStore.readEntries({ signal: "files", since: updateId })) { - await saveContentToDomainStore(uri, extract(await deps.files.read(uri))); - await deps.updatesStore.setUpdate({ signal: "content", uri, stamp: transactionId }); - } - return true; // only on full completion → TransactionStore advances → resumable - }; -} -``` - -**2. Fine, per-cell handled watermark** (`readUpdates` / `readCellUpdates` + `handleUpdate`). The watermark is per `(signal, cell, uri)`. Use this when **multiple cells consume the same signal and must handle each change independently**, or for **sink/fan-out cells** that have no single output signal to act as their watermark. Each cell advances its own watermark by calling `handleUpdate` on the input it just processed: - -```ts -import { readCellUpdates } from "@statewalker/shared-dataflow"; - -// Two cells both consume "file-source"; each tracks the same file independently. -function newPreviewer(deps: { graph: DataflowGraph; updatesStore: UpdatesStore }): CellHandler { - const CELL = "Previewer"; - return async () => { - for await (const entry of readCellUpdates(deps.updatesStore, deps.graph, CELL)) { - const changed = await renderPreview(entry.uri); // false if output unchanged - // Advance the watermark on EVERY observed uri — work, skip, or throw — - // so a stuck item can't loop forever and the cascade converges. - await deps.updatesStore.handleUpdate({ - signal: entry.signal, uri: entry.uri, cell: CELL, stamp: entry.stamp, - }); - // Announce downstream ONLY when output actually changed, propagating the - // observed stamp. Skips do NOT re-stamp the output → no spurious cascade. - if (changed) { - await deps.updatesStore.setUpdate({ signal: "preview", uri: entry.uri, stamp: entry.stamp }); - } - } - return true; - }; -} -``` - -> **Watermark vs. output, decoupled.** In the fine model, "I consumed my input" (`handleUpdate`, always) is a separate fact from "I produced an output" (`setUpdate`, only on real change). Conflating them — advancing the downstream signal on every observed uri, including no-op skips — makes unchanged content ripple needlessly through the rest of the graph. Keep them apart. - -### `readCellUpdates` + `aggregateByUri` — graph-aware per-cell diff - -`readCellUpdates(store, graph, cellId, { uriPrefix? })` is the fine-model reader. It discovers the cell's input signals via `graph.getCellInputs(cellId)` and, for each, opens `readUpdates({ signal, cell: cellId, orderBy: "uri" })`, then merges them with a streaming k-way URI merge. - -- The watermark dimension is the **`cellId` itself** — not any output signal — so **sink cells (inputs, no outputs) work** and **probers (no inputs) yield nothing**. -- Output is **URI-ascending**. A uri fresh on N of the cell's input signals appears N times; same-uri entries are emitted adjacently in `graph.getCellInputs` declaration order, so a consumer can collapse per-uri in one forward pass. -- Memory is O(number of input signals), independent of how many URIs match — and the merge is lazy, so a consumer that `break`s early stops the underlying reads. - -```ts -import { readCellUpdates, aggregateByUri } from "@statewalker/shared-dataflow"; - -// One record per uri, with every contributing upstream entry: -const byUri = await aggregateByUri(readCellUpdates(store, graph, "Index")); -for (const [uri, entries] of byUri) { - // entries = the fresh updates across this cell's inputs for `uri` -} -``` - -After handling a yielded entry, call `handleUpdate` with `stamp >= entry.stamp` to advance the watermark; otherwise the uri reappears next call. To force a full re-run of one cell (and, via re-stamped outputs, its downstream), call `clearHandled` on each of its input signals. - -### `InMemoryUpdatesStore` - -Reference implementation backed by two maps — `Map>` (updates) and `Map>>` (handled). State lives in this process; nothing persists across restarts. The constructor accepts an optional serialized state, and `snapshot()` / `toJSON()` dump it: - -```ts -import { InMemoryUpdatesStore } from "@statewalker/shared-dataflow"; - -const store = new InMemoryUpdatesStore(); -await store.setUpdate({ signal: "files", uri: "f1", stamp: 1 }); -await store.handleUpdate({ signal: "files", uri: "f1", cell: "Extractor", stamp: 1 }); - -// Round-trip via JSON (both relations survive): -const restored = new InMemoryUpdatesStore(JSON.parse(JSON.stringify(store))); -``` - -The serialized shape is a JSON-safe object with both relations: - -```ts -type SerializedUpdatesStore = { - updates: { [signal: string]: { [uri: string]: number } }; - handled: { [signal: string]: { [cell: string]: { [uri: string]: number } } }; -}; -``` - -Both directions are defensively copied: the store never holds a live reference to caller-provided objects, and a returned snapshot can be mutated freely. For backward compatibility the constructor also accepts a **legacy flat** `{ [signal]: { [uri]: stamp } }` object (no `updates`/`handled` keys), loading it as the `updates` relation with empty handled state. - -This serialized shape is private to `InMemoryUpdatesStore` (and file-backed wrappers that reuse it) — it is **not** part of the `UpdatesStore` interface. - -### Storage-agnostic — maps onto a database - -The `UpdatesStore` interface references no key encoding, separator, or serialization, so it translates directly onto two relational tables: - -```sql -CREATE TABLE updates (signal TEXT, uri TEXT, stamp BIGINT, PRIMARY KEY (signal, uri)); -CREATE TABLE handled (signal TEXT, cell TEXT, uri TEXT, stamp BIGINT, - PRIMARY KEY (signal, cell, uri), - FOREIGN KEY (signal, uri) REFERENCES updates(signal, uri) ON DELETE CASCADE); -``` - -`readUpdates` is an indexed left-join (`updates LEFT JOIN handled … WHERE u.stamp > COALESCE(h.stamp, 0)`); `removeUpdate`'s cascade is the foreign key's `ON DELETE CASCADE` (free, not the in-memory O(cells) loop); `clearHandled` is `DELETE FROM handled WHERE signal=? AND cell=?`. Nothing in the interface requires loading a full snapshot, so a DB backend needs no `snapshot()`/`toJSON()`. - -### End-to-end scenario — scanner + cascade + re-indexing - -A worked example of the **coarse model** (per-cell transaction watermark). A typical pipeline starts with a *scanner* cell. The scanner observes some external source (a files map, a directory, an inbox), detects what changed since its last visit, and publishes the changes onto a domain signal. Downstream cells transform, derive, embed, index — each one reading from one signal and writing to another, all coordinated through the same `UpdatesStore`. - -``` - scan - │ - ▼ - [ScanFiles] - │ - ▼ - files ─────────────────────┐ - │ │ - ▼ │ - [ExtractContent] │ - │ │ - ▼ │ - content │ - │ │ - ▼ │ - [SplitContent] │ - │ │ - ▼ │ - chunks ────────────────────┤ - │ │ - ▼ │ - [EmbedChunks] │ - │ │ - ▼ │ - embeddings ───────────────────┤ - │ - ▼ - [Index] -``` - -`[PascalCase]` boxes are cells; kebab-case bare names are signals. Note the fan-out — `files` feeds both `ExtractContent` and `Index`, `chunks` feeds both `EmbedChunks` and `Index` — and the fan-in: `Index` only fires after `files`, `chunks`, *and* `embeddings` have all settled for this activation (barrier semantics). - -```ts -const graph = new DataflowGraph([ - { id: "ScanFiles", inputs: ["scan"], outputs: ["files"] }, - { id: "ExtractContent", inputs: ["files"], outputs: ["content"] }, - { id: "SplitContent", inputs: ["content"], outputs: ["chunks"] }, - { id: "EmbedChunks", inputs: ["chunks"], outputs: ["embeddings"] }, - { id: "Index", inputs: ["files", "chunks", "embeddings"], outputs: [] }, -]); -``` - -`ScanFiles` is responsible for tracking per-source change markers itself — for example, comparing each file's `updatedAt` against the last value it observed for that URI — and emitting `{ signal: "files", uri, stamp: transactionId }` only for files that actually changed: - -```ts -function newFilesScanner(deps: { files: Map; updatesStore: UpdatesStore }): CellHandler { - const lastSeen = new Map(); - return async ({ transactionId }) => { - for (const [uri, file] of deps.files) { - if (file.updatedAt > (lastSeen.get(uri) ?? 0)) { - await deps.updatesStore.setUpdate({ signal: "files", uri, stamp: transactionId }); - lastSeen.set(uri, file.updatedAt); - } - } - return true; - }; -} -``` - -**Initial pass.** `manager.exec({ signals: ["scan"] })` allocates a fresh `transactionId`, walks the graph in topological order, and lets each cell read its inputs through `UpdatesStore`. `ScanFiles` publishes new `files` entries; `ExtractContent` reads them, writes to its content store and publishes `content` entries; `SplitContent` reads `content`, publishes `chunks`; `EmbedChunks` reads `chunks`, publishes `embeddings`; `Index` reads all three and updates its index. By the end of the run every cell's recorded transaction has advanced. - -**Re-indexing.** When a file changes on disk, the caller mutates the source (`files.set("f1", { body: "...", updatedAt: 2 })`) and runs `manager.exec({ signals: ["scan"] })` again. `ScanFiles` notices the bumped `updatedAt` and re-emits `{ signal: "files", uri: "f1", stamp: tx2 }`. Because `UpdatesStore` upserts by `(signal, uri)`, the row's stamp moves from `tx1` to `tx2`. Every downstream cell's next `readEntries({ signal, since: updateId })` query (where `updateId` is the cell's last recorded tx, less than `tx2`) yields the URI again, and the cell re-processes it. Re-indexing falls out of the contract — there is no special "re-index" code path. - -**No-op re-runs.** If nothing changed (no file's `updatedAt` advanced), `ScanFiles` emits nothing, every downstream cell reads zero entries, and the cascade is a no-op. Idempotence is structural. - -> The same pipeline written in the **fine model** would swap `readEntries({ signal, since: updateId })` for `readCellUpdates(store, graph, cellId)` and the trailing `setUpdate(output)` for `handleUpdate(input)` + a conditional `setUpdate(output)` — needed once two cells consume the same signal independently (e.g. an `Index` and a `Previewer` both reading `files`). - -### Deletion — tombstone signals + `removeUpdate` - -Deletion is propagated as its own signal (a convention, not a contract). When a file disappears, the upstream emits `{ signal: "files:removed", uri }`; downstream cells declare `"files:removed"` (or whatever naming you prefer — `"-files"`, `"files-deleted"`) as an input and react accordingly. The graph's topological order fans the deletion through the cascade just like a creation. - -When a tombstone-consuming handler has finished propagating the deletion to its own downstream stores, it cleans up the upstream pair via `removeUpdate` — both the original `"files"` row and the consumed `"files:removed"` row for that URI — so the next sweep does not re-process the same deletion. Because `removeUpdate` cascades into the handled relation, **every cell's watermark for that URI is cleared too**, so a later re-created URI is seen fresh by every consumer. The store enforces nothing about signal naming. - -### Caller responsibilities - -Things the store deliberately does NOT enforce: - -- **Stamp discipline.** Stamps are caller-supplied; the store never derives, validates (beyond finiteness), or compares them across calls. In the coarse model pass the activation's `transactionId`; in the fine model pass the observed upstream `entry.stamp`. -- **Signal & cell naming.** Any string is a valid `signal` or `cell` (spaces and delimiters included) — the store reserves no characters. It does not know which signals a `DataflowGraph` declares; a handler that writes to a signal its cell did not declare as an `output` is a topology bug invisible to the store. -- **Tombstone naming convention.** The `:removed` (or whatever) convention is yours to set, recorded in your graph topology. - -## Updates manager - -`UpdatesManager` is the runtime that drives handler execution over the graph using a `TransactionStore`. It exposes two methods: - -- **`run(seeds?)`** — an async generator that yields `StageInfo` events. The caller can drive the activation one stage at a time, pausing between cells. -- **`exec(seeds?)`** — convenience: iterates `run` to completion and resolves. Use when you don't need per-stage observation. - -```ts -import { - DataflowGraph, - InMemoryTransactionStore, - UpdatesManager, -} from "@statewalker/shared-dataflow"; - -const graph = new DataflowGraph([ - { id: "Detect", inputs: ["fs-tick"], outputs: ["files-changed"] }, - { id: "Extract", inputs: ["files-changed"], outputs: ["extracted"] }, - { id: "Chunk", inputs: ["extracted"], outputs: ["chunks"] }, -]); -const store = new InMemoryTransactionStore(); - -const manager = new UpdatesManager({ - graph, - store, - handlers: { - Detect: async ({ updateId, transactionId }) => { /* ... */ return true; }, - Extract: async ({ updateId, transactionId }) => { /* ... */ return true; }, - Chunk: async ({ updateId, transactionId }) => { /* ... */ return true; }, - }, - onError: (cellId, error) => console.error(`[${cellId}]`, error), -}); - -// External trigger (e.g. fs-watcher fires) — convenience form, drain to completion. -await manager.exec({ signals: ["fs-tick"] }); - -// Periodic sweep — runs all probers (cells with inputs: []) plus their cascade. -await manager.exec(); -``` - -### Seeds — signals, cells, or none - -The argument to `run` / `exec` is a discriminated union: - -- `{ signals: Iterable }` — start from changed signals. The cells consuming them and their downstream cascade run, in topological order. -- `{ cells: Iterable }` — start from explicit cell ids. Those cells plus their downstream cascade run. Used to resume an interrupted activation (see "Restart" below). -- Omitted — run probers (cells with `inputs: []`) and everything they cascade into. - -The two seed forms are mutually exclusive; mixing them is a type error. - -### Per-activation lifecycle - -Per call to `run()` / `exec()`: - -1. A new `transactionId` is allocated via `store.newTransactionId()`. **All cells in this activation share it.** -2. The cell list is computed from the seeds (or probers when omitted). -3. Each cell's handler is invoked with `{ updateId: store.getCellTransaction(cellId), transactionId }`. -4. On `true` → `store.setCellTransaction(cellId, transactionId)`. On `false` or thrown → store untouched; thrown errors are forwarded to `onError`. - -Activations are serialized. The in-flight guard is set when iteration begins (first `next()`) and cleared when the generator finishes or is closed. A second `run` whose iteration begins while another is still in progress throws. - -### Stage events — observing the activation - -`run` yields `StageInfo` events as the activation progresses: - -```ts -type StageInfo = - | { type: "begin"; transactionId: number } - | { type: "end"; transactionId: number } - | { - type: "call"; - transactionId: number; - cellId: CellId; - updateId: number; // the cell's prior successful tx, passed to its handler - result: boolean; // true = handler finished, false = handler returned false or threw - }; -``` - -Exactly one `begin`, one `call` per executed cell in topological order, one `end`. All three carry the same `transactionId`. - -Stepping the generator yourself lets you (a) checkpoint progress to disk between cells, (b) pause until external state catches up, or (c) abort early: - -```ts -const it = manager.run({ signals: ["fs-tick"] }); -for await (const stage of it) { - if (stage.type === "call" && stage.cellId === "Extract" && !stage.result) { - // Extract failed — checkpoint and bail out; the generator's `finally` - // releases the in-flight guard so the next `run` / `exec` can start. - await it.return(undefined); - break; - } -} -``` - -### Restart — finalize interrupted cells before the next sweep - -When a handler returns `false`, its cell's `TransactionStore` entry does not advance — the cell will re-process the same upstream entries on the next activation. But the next activation usually starts from a *new* upstream change (e.g., the periodic scan). If you want to **finish the previous round** before introducing new work, collect the failed cell ids and pass them back as `{ cells }`: - -```ts -const incompleteCells: CellId[] = []; -for await (const stage of manager.run({ signals: ["scan"] })) { - if (stage.type === "call" && !stage.result) incompleteCells.push(stage.cellId); -} - -if (incompleteCells.length > 0) { - // Finalize last round's interrupted cells + their downstream cascade. - // Each cell's handler reads with `since: updateId` (still its last - // successful tx, before this round) and picks up exactly where it left off. - await manager.exec({ cells: incompleteCells }); -} - -// Now safe to start the next scan — earlier upstream changes have settled. -await manager.exec({ signals: ["scan"] }); -``` - -This pattern is useful when handlers process upstream entries in batches (returning `false` to signal "more to do"): the operator can drain the pipeline before scanning again, avoiding pile-up. - -### Handler contract - -```ts -type CellHandler = (params: { - updateId: number; // = lastSuccessTx for this cell, or 0 - transactionId: number; // = activation's tx -}) => Promise; -``` - -Handlers are expected to be **idempotent** — they may be re-invoked with the same `updateId` after a previous failure. Coordinate per-entry changes between handlers through the [`UpdatesStore`](#updates-store) above, in either the coarse model (read with `since: updateId`, write with `stamp: transactionId`) or the fine model (read with `readCellUpdates` / `readUpdates`, advance with `handleUpdate`, write outputs only on real change). Both make replays skip work that already published. - -## License - -MIT. diff --git a/packages/shared-dataflow/docs/use-cases.md b/packages/shared-dataflow/docs/use-cases.md deleted file mode 100644 index 30aeddb..0000000 --- a/packages/shared-dataflow/docs/use-cases.md +++ /dev/null @@ -1,20 +0,0 @@ -# Use cases - -`shared-dataflow` was born from content pipelines, but the shape is general. Anything that can be drawn as "boxes consume/produce named channels, and change on the left should ripple to the right" maps onto it. - -The common signature: **stages where each one cares only about "what's new since I last looked," and where correctness depends on the cascade settling into a consistent state**. If you can draw that diagram on a whiteboard, `shared-dataflow` is the runtime under it. - -## Example domains - -- **Content ingestion / RAG.** Scan files → extract text → chunk → embed → write to vector index. Re-embed only what changed; delete only what disappeared. The canonical example in the [README](../README.md). -- **ETL / data warehouse builds.** Raw tables → staging models → marts → exposures. Like a minimal in-process [dbt](https://www.getdbt.com/), but driven by per-row stamps instead of full table rebuilds. A small change in `orders` only refreshes the downstream models that depend on it, in the right order. -- **CI / build pipelines.** Source change → compile → unit test → integration test → bundle → deploy. The dependency graph between build artifacts is exactly a dataflow graph; the per-cell `updateId` is the cache key. Resumability lets a flaky integration test re-run without re-compiling. -- **Devops / configuration cascades.** Config repo change → render manifests → push to cluster → restart dependent services → run smoke tests. Each step a cell, each artifact a signal. Tombstones handle "service removed from config." -- **Cache / materialized view invalidation.** Source row updated → recompute derived cache → bust HTTP cache → notify subscribers. The "what changed since I last checked?" query is exactly `readEntries({ signal, since })`. -- **Reactive computation engines.** Spreadsheet-style "cell A changed, recompute everything downstream" — but distributed, asynchronous, and persistent. Useful for notebooks, dashboards, or live derivations where a full re-eval is too expensive. -- **IoT / telemetry pipelines.** Sensor reading arrives → aggregate to a window → check thresholds → publish alert. Each device is a URI; the scanner is the ingest gateway; downstream cells maintain rollups and alarm state. -- **ML feature pipelines.** Raw events → feature tables → training set → trained model → evaluation. Retrain only when the upstream features actually moved; never silently skip a stage. -- **Static site generators.** Markdown change → re-render page → rebuild index → invalidate CDN. The same pattern dressed up as a website. -- **Backup / replication / sync.** Source store → checksum → diff → push to mirror. The mirror's last-success tx tells you exactly the slice to ship. -- **Search-index maintenance.** Document changes → tokenize → update inverted index → update facets. Adding a new derived index is one more cell, not a re-architecture. -- **Compliance / audit pipelines.** Event log → classify → redact → archive → notify. Each row processed exactly once, replayable from any point, with the bookmark proving how far you got. diff --git a/packages/shared-dataflow/package.json b/packages/shared-dataflow/package.json deleted file mode 100644 index 80d9a34..0000000 --- a/packages/shared-dataflow/package.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "@statewalker/shared-dataflow", - "version": "0.1.0", - "private": false, - "type": "module", - "description": "Signal-driven dataflow graph (forward impact propagation + filtered Kahn topo sort), with per-cell transaction and per-entry updates stores plus in-memory implementations. Zero deps.", - "homepage": "https://github.com/statewalker/statewalker-shared", - "author": { - "name": "Mikhail Kotelnikov", - "email": "mikhail.kotelnikov@gmail.com" - }, - "license": "MIT", - "repository": { - "type": "git", - "url": "git+ssh://git@github.com/statewalker/statewalker-shared.git" - }, - "exports": { - ".": "./src/index.ts" - }, - "files": [ - "dist", - "src" - ], - "scripts": { - "build": "tsdown", - "dev": "tsdown --watch", - "test": "vitest run", - "test:watch": "vitest", - "typecheck": "tsc --noEmit", - "clean": "rimraf dist", - "lint": "biome check --write .", - "format": "biome format --write ." - }, - "devDependencies": { - "rimraf": "catalog:", - "tsdown": "catalog:", - "typescript": "catalog:", - "vitest": "catalog:" - }, - "sideEffects": false, - "publishConfig": { - "access": "public" - } -} diff --git a/packages/shared-dataflow/src/dataflow-graph.ts b/packages/shared-dataflow/src/dataflow-graph.ts deleted file mode 100644 index e770b03..0000000 --- a/packages/shared-dataflow/src/dataflow-graph.ts +++ /dev/null @@ -1,217 +0,0 @@ -import type { CellDefinition, CellId, Signal } from "./types.js"; - -/** - * Signal-driven dataflow graph. - * - * A cell declares the signals it reads (`inputs`) and the signals it writes - * (`outputs`). Multiple producers per signal are allowed. Given a set of - * changed signals, `getExecutionOrder` returns the cells that must run, in a - * valid topological order, under barrier semantics: - * - * "A consumer must run after ALL producers of its inputs that are - * themselves scheduled in this execution." - */ -export class DataflowGraph { - private readonly cells = new Map(); - private readonly signalToConsumers = new Map>(); - private readonly signalToProducers = new Map>(); - - constructor(cellDefs: readonly CellDefinition[]) { - for (const cell of cellDefs) { - if (this.cells.has(cell.id)) { - throw new Error(`Duplicate cell id: ${cell.id}`); - } - this.cells.set(cell.id, cell); - - for (const input of cell.inputs) { - addToSetMap(this.signalToConsumers, input, cell.id); - } - for (const output of cell.outputs) { - addToSetMap(this.signalToProducers, output, cell.id); - } - } - } - - getAllCells(): CellId[] { - return [...this.cells.keys()]; - } - - getCellInputs(cellId: CellId): Signal[] { - return [...(this.cells.get(cellId)?.inputs ?? [])]; - } - - getCellOutputs(cellId: CellId): Signal[] { - return [...(this.cells.get(cellId)?.outputs ?? [])]; - } - - getCellsConsuming(signal: Signal): Set { - return new Set(this.signalToConsumers.get(signal) ?? []); - } - - getCellsProducing(signal: Signal): Set { - return new Set(this.signalToProducers.get(signal) ?? []); - } - - /** - * Main API. Returns the impacted cells in a valid execution order. - * Throws if the impacted subgraph contains a cycle. - */ - getExecutionOrder(changedSignals: Iterable): CellId[] { - const seeds = this.findSeedCells(changedSignals); - const impacted = this.propagateDownstream(seeds); - return this.topoSort(impacted); - } - - /** - * Same as `getExecutionOrder`, but seeded with explicit cell ids instead of - * signals. The seeded cells are included in the impacted set, then forward - * propagation walks `cell.outputs → consumers` from them as usual, and the - * combined set is topologically sorted. - * - * Use when restarting a previously-interrupted activation: pass the cells - * whose handlers returned `false` last time, and the manager will run those - * cells plus any downstream consumers their outputs reach. - * - * Cell ids not present in this graph are silently dropped. - */ - getExecutionOrderFromCells(startCells: Iterable): CellId[] { - const seeds = new Set(); - for (const cellId of startCells) { - if (this.cells.has(cellId)) seeds.add(cellId); - } - const impacted = this.propagateDownstream(seeds); - return this.topoSort(impacted); - } - - // -- Step 1: cells that directly consume a changed signal ---------------- - - private findSeedCells(signals: Iterable): Set { - const seeds = new Set(); - for (const signal of signals) { - const consumers = this.signalToConsumers.get(signal); - if (!consumers) continue; - for (const cellId of consumers) seeds.add(cellId); - } - return seeds; - } - - // -- Step 2: forward BFS through outputs → consumers --------------------- - - private propagateDownstream(seeds: Set): Set { - const impacted = new Set(seeds); - const queue: CellId[] = [...seeds]; - let head = 0; - - while (head < queue.length) { - const cellId = queue[head++] as CellId; - const cell = this.cells.get(cellId); - if (!cell) continue; - - for (const output of cell.outputs) { - const consumers = this.signalToConsumers.get(output); - if (!consumers) continue; - for (const next of consumers) { - if (!impacted.has(next)) { - impacted.add(next); - queue.push(next); - } - } - } - } - return impacted; - } - - // -- Step 3: filtered Kahn's algorithm on the impacted subgraph ---------- - - private topoSort(impacted: Set): CellId[] { - // deps[A] = producers (within `impacted`) that A depends on - // reverse[B] = cells (within `impacted`) that depend on B - const deps = new Map>(); - const reverse = new Map>(); - - for (const cellId of impacted) { - const cell = this.cells.get(cellId); - if (!cell) continue; - const cellDeps = new Set(); - - for (const input of cell.inputs) { - const producers = this.signalToProducers.get(input); - if (!producers) continue; - for (const producer of producers) { - if (producer !== cellId && impacted.has(producer)) { - cellDeps.add(producer); - addToSetMap(reverse, producer, cellId); - } - } - } - deps.set(cellId, cellDeps); - } - - const inDegree = new Map(); - for (const cellId of impacted) { - inDegree.set(cellId, deps.get(cellId)?.size ?? 0); - } - - const queue: CellId[] = []; - for (const [cellId, deg] of inDegree) { - if (deg === 0) queue.push(cellId); - } - - const result: CellId[] = []; - let head = 0; - while (head < queue.length) { - const current = queue[head++] as CellId; - result.push(current); - - const dependents = reverse.get(current); - if (!dependents) continue; - for (const dependent of dependents) { - const deg = (inDegree.get(dependent) ?? 0) - 1; - inDegree.set(dependent, deg); - if (deg === 0) queue.push(dependent); - } - } - - if (result.length !== impacted.size) { - const processed = new Set(result); - const remaining = new Set([...impacted].filter((c) => !processed.has(c))); - const cycleMembers = [...remaining].filter((c) => reachesSelf(c, remaining, deps)); - throw new Error(`Cycle detected among cells: ${cycleMembers.join(", ")}`); - } - - return result; - } -} - -function addToSetMap(map: Map>, key: K, value: V): void { - let set = map.get(key); - if (!set) { - set = new Set(); - map.set(key, set); - } - set.add(value); -} - -/** - * True iff `start` can reach itself by walking `deps` edges restricted to - * `scope`. Used after Kahn's leaves residual nodes — only the ones in an - * actual cycle satisfy this, separating cycle members from cells that are - * merely downstream of a cycle. - */ -function reachesSelf(start: CellId, scope: Set, deps: Map>): boolean { - const visited = new Set(); - const stack: CellId[] = []; - for (const d of deps.get(start) ?? []) { - if (scope.has(d)) stack.push(d); - } - while (stack.length > 0) { - const cur = stack.pop() as CellId; - if (cur === start) return true; - if (visited.has(cur)) continue; - visited.add(cur); - for (const d of deps.get(cur) ?? []) { - if (scope.has(d)) stack.push(d); - } - } - return false; -} diff --git a/packages/shared-dataflow/src/in-memory-transaction-store.ts b/packages/shared-dataflow/src/in-memory-transaction-store.ts deleted file mode 100644 index d80d266..0000000 --- a/packages/shared-dataflow/src/in-memory-transaction-store.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { TransactionStore } from "./transaction-store.js"; -import type { CellId } from "./types.js"; - -/** - * In-memory `TransactionStore`. State lives in this process; nothing is - * persisted across restarts. Suitable for tests and single-process use. - */ -export class InMemoryTransactionStore implements TransactionStore { - private nextTx = 1; - private readonly cellTransactions = new Map(); - - async newTransactionId(): Promise { - return this.nextTx++; - } - - async setCellTransaction(cell: CellId, transactionId: number): Promise { - this.cellTransactions.set(cell, transactionId); - } - - async getCellTransaction(cell: CellId): Promise { - return this.cellTransactions.get(cell) ?? 0; - } - - async *getCellsTransactions(sinceTransactionId?: number): AsyncGenerator<[CellId, number]> { - for (const [cell, tx] of this.cellTransactions) { - if (sinceTransactionId === undefined || tx > sinceTransactionId) { - yield [cell, tx]; - } - } - } - - async removeCellTransactions(cell: CellId): Promise { - this.cellTransactions.delete(cell); - } -} diff --git a/packages/shared-dataflow/src/in-memory-updates-store.ts b/packages/shared-dataflow/src/in-memory-updates-store.ts deleted file mode 100644 index 3194a4d..0000000 --- a/packages/shared-dataflow/src/in-memory-updates-store.ts +++ /dev/null @@ -1,233 +0,0 @@ -import type { Signal } from "./types.js"; -import type { - HandledEntry, - ReadOrderBy, - SerializedUpdatesStore, - UpdateEntry, - UpdatesStore, -} from "./updates-store.js"; - -function compareMatches(orderBy: ReadOrderBy | undefined) { - if (orderBy === "uri") { - return (a: [string, number], b: [string, number]) => { - if (a[0] === b[0]) return 0; - return a[0] < b[0] ? -1 : 1; - }; - } - return (a: [string, number], b: [string, number]) => a[1] - b[1]; -} - -function assertFiniteStamp(stamp: number, method: string): void { - if (!Number.isFinite(stamp)) { - throw new Error(`InMemoryUpdatesStore.${method}: stamp must be a finite number, got ${stamp}`); - } -} - -/** - * `out[key] = value` invokes the `__proto__` setter for that key, silently - * dropping entries whose key is "__proto__". `defineProperty` stores any - * string as an own enumerable property so a snapshot/restore round-trip is - * lossless and JSON-safe. - */ -function defineEnumerable(obj: Record, key: string, value: T): void { - Object.defineProperty(obj, key, { - value, - enumerable: true, - configurable: true, - writable: true, - }); -} - -function isLegacyFlat(state: object): boolean { - // The two-relation shape always carries an `updates` own-property. Anything - // without it is a legacy flat `{ [signal]: { [uri]: number } }` snapshot. - return !Object.hasOwn(state, "updates") && !Object.hasOwn(state, "handled"); -} - -/** - * In-memory `UpdatesStore` reference implementation. State lives in this - * process; nothing is persisted across restarts. Suitable for tests and - * single-process pipelines. - * - * State is held as two maps mirroring the two logical relations: - * `updates: Map>` and - * `handled: Map>>`. - * - * The constructor accepts (and `snapshot()` / `toJSON()` return) a - * `SerializedUpdatesStore` — a plain JSON-safe object `{ updates, handled }`. - * A legacy flat `{ [signal]: { [uri]: number } }` object is also accepted and - * loaded as the `updates` relation (empty handled state). Both directions are - * defensively copied: the store never holds a live reference to caller-provided - * objects, and the snapshot returned to a caller can be mutated freely. - */ -export class InMemoryUpdatesStore implements UpdatesStore { - private readonly updates = new Map>(); - private readonly handled = new Map>>(); - - constructor(initialState?: SerializedUpdatesStore) { - if (!initialState) return; - if (isLegacyFlat(initialState)) { - const flat = initialState as unknown as { [signal: string]: { [uri: string]: number } }; - for (const [signal, rows] of Object.entries(flat)) { - this.loadUpdates(signal, rows); - } - return; - } - for (const [signal, rows] of Object.entries(initialState.updates ?? {})) { - this.loadUpdates(signal, rows); - } - for (const [signal, cells] of Object.entries(initialState.handled ?? {})) { - for (const [cell, rows] of Object.entries(cells)) { - for (const [uri, stamp] of Object.entries(rows)) { - this.handledInner(signal, cell).set(uri, stamp); - } - } - } - } - - private loadUpdates(signal: string, rows: { [uri: string]: number }): void { - const inner = new Map(); - for (const [uri, stamp] of Object.entries(rows)) inner.set(uri, stamp); - if (inner.size > 0) this.updates.set(signal, inner); - } - - private handledInner(signal: Signal, cell: string): Map { - let cells = this.handled.get(signal); - if (!cells) { - cells = new Map>(); - this.handled.set(signal, cells); - } - let inner = cells.get(cell); - if (!inner) { - inner = new Map(); - cells.set(cell, inner); - } - return inner; - } - - async setUpdate(entry: UpdateEntry): Promise { - assertFiniteStamp(entry.stamp, "setUpdate"); - let inner = this.updates.get(entry.signal); - if (!inner) { - inner = new Map(); - this.updates.set(entry.signal, inner); - } - inner.set(entry.uri, entry.stamp); - } - - async setUpdates(entries: ReadonlyArray): Promise { - for (const entry of entries) await this.setUpdate(entry); - } - - async handleUpdate(entry: HandledEntry): Promise { - assertFiniteStamp(entry.stamp, "handleUpdate"); - this.handledInner(entry.signal, entry.cell).set(entry.uri, entry.stamp); - } - - async handleUpdates(entries: ReadonlyArray): Promise { - for (const entry of entries) await this.handleUpdate(entry); - } - - async clearHandled(key: { signal: Signal; cell: string }): Promise { - const cells = this.handled.get(key.signal); - const inner = cells?.get(key.cell); - if (!cells || !inner) return 0; - const removed = inner.size; - cells.delete(key.cell); - if (cells.size === 0) this.handled.delete(key.signal); - return removed; - } - - async removeUpdate(key: { signal: Signal; uri: string }): Promise { - const inner = this.updates.get(key.signal); - if (inner) { - inner.delete(key.uri); - if (inner.size === 0) this.updates.delete(key.signal); - } - // Cascade: drop this URI from every cell's handled map for the signal. - const cells = this.handled.get(key.signal); - if (cells) { - for (const [cell, uris] of cells) { - uris.delete(key.uri); - if (uris.size === 0) cells.delete(cell); - } - if (cells.size === 0) this.handled.delete(key.signal); - } - } - - async removeUpdates(keys: ReadonlyArray<{ signal: Signal; uri: string }>): Promise { - for (const key of keys) await this.removeUpdate(key); - } - - async *readEntries(opts: { - signal: Signal; - since: number; - uriPrefix?: string; - orderBy?: ReadOrderBy; - }): AsyncIterable { - const inner = this.updates.get(opts.signal); - if (!inner) return; - const prefix = opts.uriPrefix ?? ""; - const matches: Array<[string, number]> = []; - for (const [uri, stamp] of inner) { - if (stamp > opts.since && (prefix === "" || uri.startsWith(prefix))) { - matches.push([uri, stamp]); - } - } - matches.sort(compareMatches(opts.orderBy)); - for (const [uri, stamp] of matches) { - yield { signal: opts.signal, uri, stamp }; - } - } - - async *readUpdates(opts: { - signal: Signal; - cell: string; - uriPrefix?: string; - orderBy?: ReadOrderBy; - }): AsyncIterable { - const inner = this.updates.get(opts.signal); - if (!inner) return; - const handledByCell = this.handled.get(opts.signal)?.get(opts.cell); - const prefix = opts.uriPrefix ?? ""; - const matches: Array<[string, number]> = []; - for (const [uri, stamp] of inner) { - if (prefix !== "" && !uri.startsWith(prefix)) continue; - const handledStamp = handledByCell?.get(uri) ?? 0; - if (stamp > handledStamp) matches.push([uri, stamp]); - } - matches.sort(compareMatches(opts.orderBy)); - for (const [uri, stamp] of matches) { - yield { signal: opts.signal, uri, stamp }; - } - } - - /** - * Return a fresh JSON-safe snapshot of both relations. Mutating the - * returned object does not affect the store. - */ - snapshot(): SerializedUpdatesStore { - const updates = {} as SerializedUpdatesStore["updates"]; - for (const [signal, inner] of this.updates) { - const rows = {} as { [uri: string]: number }; - for (const [uri, stamp] of inner) defineEnumerable(rows, uri, stamp); - defineEnumerable(updates, signal, rows); - } - const handled = {} as SerializedUpdatesStore["handled"]; - for (const [signal, cells] of this.handled) { - const cellsOut = {} as { [cell: string]: { [uri: string]: number } }; - for (const [cell, inner] of cells) { - const rows = {} as { [uri: string]: number }; - for (const [uri, stamp] of inner) defineEnumerable(rows, uri, stamp); - defineEnumerable(cellsOut, cell, rows); - } - defineEnumerable(handled, signal, cellsOut); - } - return { updates, handled }; - } - - /** Makes `JSON.stringify(store)` produce a valid `SerializedUpdatesStore` JSON string. */ - toJSON(): SerializedUpdatesStore { - return this.snapshot(); - } -} diff --git a/packages/shared-dataflow/src/index.ts b/packages/shared-dataflow/src/index.ts deleted file mode 100644 index a5a22f5..0000000 --- a/packages/shared-dataflow/src/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -export { DataflowGraph } from "./dataflow-graph.js"; -export { InMemoryTransactionStore } from "./in-memory-transaction-store.js"; -export { InMemoryUpdatesStore } from "./in-memory-updates-store.js"; -export { - aggregateByUri, - readCellUpdates, -} from "./read-cell-updates.js"; -export type { TransactionStore } from "./transaction-store.js"; -export type { CellDefinition, CellId, Signal } from "./types.js"; -export type { - CellHandler, - RunSeeds, - StageInfo, - UpdatesManagerOptions, -} from "./updates-manager.js"; -export { UpdatesManager } from "./updates-manager.js"; -export type { - HandledEntry, - ReadOrderBy, - SerializedUpdatesStore, - UpdateEntry, - UpdatesStore, -} from "./updates-store.js"; diff --git a/packages/shared-dataflow/src/read-cell-updates.ts b/packages/shared-dataflow/src/read-cell-updates.ts deleted file mode 100644 index 748d802..0000000 --- a/packages/shared-dataflow/src/read-cell-updates.ts +++ /dev/null @@ -1,123 +0,0 @@ -import type { DataflowGraph } from "./dataflow-graph.js"; -import type { CellId } from "./types.js"; -import type { UpdateEntry, UpdatesStore } from "./updates-store.js"; - -/** - * Yield all updates on the cell's upstream signals that the cell hasn't - * handled yet. Discovers the cell's upstream signals via - * `graph.getCellInputs(cellId)`; the per-URI watermark is keyed on the - * `cellId` itself (the cell's `handled` rows), NOT on any output signal — - * so a cell needs no output to track its progress. - * - * Opens one URI-ordered diff stream per upstream signal (via - * `readUpdates({ cell: cellId, orderBy: "uri" })`) and merges them with a - * streaming k-way URI merge. Memory is O(M) where M is the number of - * upstream signals — independent of the number of matching URIs. - * - * Output order is URI-ascending. When the same URI appears in multiple - * upstream signals, the entries are emitted adjacently in the order the - * upstream signals were declared in `graph.getCellInputs(cellId)`, so - * consumers can collapse per-URI in one forward pass without buffering. - * - * A URI updated by N upstream signals appears N times (once per upstream - * entry). Use `aggregateByUri` if the consumer wants one record per URI. - * - * Cells with no inputs (probers) yield nothing. Sink cells (inputs but no - * outputs) work normally — the cell-id watermark removes the old - * "first output signal" requirement. - */ -export async function* readCellUpdates( - store: UpdatesStore, - graph: DataflowGraph, - cellId: CellId, - opts?: { uriPrefix?: string }, -): AsyncIterable { - const inputs = graph.getCellInputs(cellId); - if (inputs.length === 0) return; - const uriPrefix = opts?.uriPrefix; - - // One URI-ordered iterator per upstream signal. - const iters: Array> = inputs.map((upstream) => { - const iterable = store.readUpdates({ - signal: upstream, - cell: cellId, - uriPrefix, - orderBy: "uri", - }); - return iterable[Symbol.asyncIterator](); - }); - - try { - // Prime the head of each stream. - const heads: Array = await Promise.all( - iters.map(async (it) => { - const r = await it.next(); - return r.done ? null : r.value; - }), - ); - - // Streaming k-way merge by URI. For URI ties, keep - // signal-declaration order by preferring the smaller index. - while (true) { - let minIdx = -1; - for (let i = 0; i < heads.length; i++) { - const h = heads[i]; - if (h === null || h === undefined) continue; - if (minIdx === -1) { - minIdx = i; - continue; - } - // biome-ignore lint/style/noNonNullAssertion: minIdx ≥ 0 implies heads[minIdx] is not null - const cur = heads[minIdx]!; - if (h.uri < cur.uri) minIdx = i; - } - if (minIdx === -1) return; - // biome-ignore lint/style/noNonNullAssertion: minIdx selected from a non-null head - const entry = heads[minIdx]!; - yield entry; - // biome-ignore lint/style/noNonNullAssertion: iters[minIdx] paired with heads[minIdx] - const next = await iters[minIdx]!.next(); - heads[minIdx] = next.done ? null : next.value; - } - } finally { - // Best-effort close of any non-exhausted iterators on early break. - await Promise.all( - iters.map(async (it) => { - if (it.return) { - try { - await it.return(undefined); - } catch { - // Swallow: a throwing iterator close must not mask the - // primary outcome of the merge. - } - } - }), - ); - } -} - -/** - * Drain a cell-updates stream into a Map keyed by URI, where each value is - * the list of upstream entries that contributed (potentially one per - * upstream signal). Useful when the cell's per-URI work depends on which - * upstream signal(s) are fresh, or simply to dedupe URIs when the cell only - * cares that "something" changed. - * - * Insertion order of the returned Map preserves encounter order in the - * source stream (so within each upstream signal, URIs are URI-ascending; - * across upstream signals, the order follows `graph.getCellInputs`). - */ -export async function aggregateByUri( - source: AsyncIterable, -): Promise> { - const out = new Map(); - for await (const entry of source) { - const bucket = out.get(entry.uri); - if (bucket) { - bucket.push(entry); - } else { - out.set(entry.uri, [entry]); - } - } - return out; -} diff --git a/packages/shared-dataflow/src/transaction-store.ts b/packages/shared-dataflow/src/transaction-store.ts deleted file mode 100644 index ab57c06..0000000 --- a/packages/shared-dataflow/src/transaction-store.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { CellId } from "./types.js"; - -/** - * Persistence-shaped interface for the bookkeeping the updates manager needs: - * a monotonic transaction-id allocator plus a per-cell record of the last - * successful transaction. - * - * Only successful runs (handler returned `true`) are recorded — failed or - * partial runs leave the cell's transaction unchanged. - * - * Implementations must guarantee: - * - * - `newTransactionId()` returns strictly increasing values across the lifetime - * of the store. Successive calls never repeat. - * - `getCellTransaction(cell)` returns `0` for a cell that has never been - * recorded (initial state). - * - `getCellsTransactions(sinceTx?)` yields all cells whose recorded - * transaction id is greater than `sinceTx` (or all recorded cells when - * `sinceTx` is omitted). Iteration order is unspecified. - */ -export interface TransactionStore { - /** Allocate a new strictly-monotonic transaction id. */ - newTransactionId(): Promise; - - /** - * Record the transaction id for a cell. Should be called only after a - * handler has returned `true` for that cell. - */ - setCellTransaction(cell: CellId, transactionId: number): Promise; - - /** - * Read the last recorded transaction id for a cell, or `0` if the cell has - * never been recorded. - */ - getCellTransaction(cell: CellId): Promise; - - /** - * Iterate over all recorded cells with their last transaction ids. When - * `sinceTransactionId` is provided, only cells with `transactionId > - * sinceTransactionId` are yielded. - */ - getCellsTransactions( - sinceTransactionId?: number, - ): AsyncGenerator<[cell: CellId, transactionId: number]>; - - /** Forget the recorded transaction id for a cell. */ - removeCellTransactions(cell: CellId): Promise; -} diff --git a/packages/shared-dataflow/src/types.ts b/packages/shared-dataflow/src/types.ts deleted file mode 100644 index a62a980..0000000 --- a/packages/shared-dataflow/src/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type CellId = string; -export type Signal = string; - -export interface CellDefinition { - id: CellId; - inputs: Signal[]; - outputs: Signal[]; -} diff --git a/packages/shared-dataflow/src/updates-manager.ts b/packages/shared-dataflow/src/updates-manager.ts deleted file mode 100644 index cfed838..0000000 --- a/packages/shared-dataflow/src/updates-manager.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { DataflowGraph } from "./dataflow-graph.js"; -import type { TransactionStore } from "./transaction-store.js"; -import type { CellId, Signal } from "./types.js"; - -export type CellHandler = (params: { - /** Tx of the most recent successful run for this cell, or `0`. */ - updateId: number; - /** Tx allocated for the current activation. Shared by all cells in this run. */ - transactionId: number; -}) => Promise; - -export interface UpdatesManagerOptions { - graph: DataflowGraph; - store: TransactionStore; - handlers: Record; - /** Called when a handler throws. The exception is otherwise swallowed. */ - onError?: (cellId: CellId, error: unknown) => void; -} - -/** - * Seeds for `run` / `exec`. Discriminated union — pick one of: - * - * - `{ signals }` — start from changed signals; the manager calls the cells - * that consume them and their downstream cascade. - * - `{ cells }` — start from explicit cell ids; the manager calls those - * cells and their downstream cascade. Used to resume a previously - * interrupted activation: pass the cells whose handlers returned `false` - * last time so the cascade finishes before the next sweep starts. - * - * Omit the argument entirely to run probers (cells with no inputs) and - * everything they cascade into. - */ -export type RunSeeds = - | { signals: Iterable; cells?: never } - | { cells: Iterable; signals?: never }; - -/** - * Stage events yielded by `run`. The caller can drive the activation one - * stage at a time, pausing between calls to inspect state or persist - * progress, and decide whether to keep iterating or close the generator. - * - * Order: exactly one `begin`, zero or more `call`s in topological order, - * exactly one `end`. All events of a single activation carry the same - * `transactionId`. - */ -export type StageInfo = - | { type: "begin"; transactionId: number } - | { type: "end"; transactionId: number } - | { - type: "call"; - transactionId: number; - cellId: CellId; - /** The cell's last successful tx — what the handler received as `updateId`. */ - updateId: number; - /** Result of the handler call: `true` = finished, `false` = interrupted (or threw). */ - result: boolean; - }; - -/** - * Drives handler execution over a `DataflowGraph`. Each activation allocates - * one transaction id, computes the topologically-ordered cell list, and - * invokes each cell's registered handler. Handlers that return `true` have - * their tx recorded; `false` and thrown exceptions leave the store untouched - * (exceptions are forwarded to `onError`). - * - * `run` is an async generator yielding `StageInfo` events. The caller can - * iterate one stage at a time, suspending the activation between cells and - * resuming it on the next `next()`. Use `exec` if you don't need that - * control — it iterates the generator to completion and returns void. - * - * Activations are serialized by an in-flight guard. The guard is set when - * generator iteration begins (first `next()`) and cleared when the generator - * finishes or is closed. A second `run()` whose iteration begins while - * another generator is still in progress throws. - */ -export class UpdatesManager { - private running = false; - - constructor(private readonly options: UpdatesManagerOptions) {} - - async *run(seeds?: RunSeeds): AsyncGenerator { - if (this.running) { - throw new Error("UpdatesManager.run is already in progress"); - } - this.running = true; - try { - const transactionId = await this.options.store.newTransactionId(); - yield { type: "begin", transactionId }; - for (const cellId of this.cellsToRun(seeds)) { - const stage = await this.executeCell(cellId, transactionId); - if (stage) yield stage; - } - yield { type: "end", transactionId }; - } finally { - this.running = false; - } - } - - /** - * Convenience: iterate `run(seeds)` to completion and resolve. Use this - * when you don't need per-stage observation. - */ - async exec(seeds?: RunSeeds): Promise { - for await (const _ of this.run(seeds)) { - // drain - } - } - - private async executeCell(cellId: CellId, transactionId: number): Promise { - const { store, handlers, onError } = this.options; - if (!Object.hasOwn(handlers, cellId)) return undefined; - const handler = handlers[cellId]; - if (!handler) return undefined; - const updateId = await store.getCellTransaction(cellId); - let result = false; - try { - result = await handler({ updateId, transactionId }); - } catch (error) { - try { - onError?.(cellId, error); - } catch { - // onError is a passive notifier; a throwing logger must not abort the run. - } - } - if (result) await store.setCellTransaction(cellId, transactionId); - return { type: "call", transactionId, cellId, updateId, result }; - } - - private cellsToRun(seeds?: RunSeeds): CellId[] { - const { graph } = this.options; - if (seeds === undefined) { - // No seeds: run all probers (cells with inputs: []) + their downstream cascade. - const probers = graph.getAllCells().filter((c) => graph.getCellInputs(c).length === 0); - if (probers.length === 0) return []; - const proberOutputs = new Set(); - for (const p of probers) { - for (const out of graph.getCellOutputs(p)) proberOutputs.add(out); - } - const downstream = graph.getExecutionOrder(proberOutputs); - // Probers have no inputs, so any order between them is valid; place them first. - return [...probers, ...downstream]; - } - if ("signals" in seeds && seeds.signals !== undefined) { - return graph.getExecutionOrder(seeds.signals); - } - if ("cells" in seeds && seeds.cells !== undefined) { - return graph.getExecutionOrderFromCells(seeds.cells); - } - return []; - } -} diff --git a/packages/shared-dataflow/src/updates-store.ts b/packages/shared-dataflow/src/updates-store.ts deleted file mode 100644 index 7aea17f..0000000 --- a/packages/shared-dataflow/src/updates-store.ts +++ /dev/null @@ -1,156 +0,0 @@ -import type { Signal } from "./types.js"; - -/** - * A single update entry: "the URI `uri` was changed on signal `signal` at - * transaction stamp `stamp`". Pure pointer — no payload. The data the URI - * addresses lives in the caller's domain store. - */ -export interface UpdateEntry { - signal: Signal; - uri: string; - stamp: number; -} - -/** - * A single per-cell handled record: "cell `cell` has caught up to the update - * `(signal, uri)` as of stamp `stamp`". The `stamp` is intended to be the - * upstream update stamp the cell observed when it handled the change. - */ -export interface HandledEntry extends UpdateEntry { - cell: string; -} - -/** - * JSON-safe serialization shape for an `UpdatesStore`. Two relations: - * - * - `updates` — outer key = signal, inner = uri → latest stamp. - * - `handled` — outer key = signal, middle = cell, inner = uri → handled stamp. - * - * Accepted by `InMemoryUpdatesStore`'s constructor and returned by - * `snapshot()` / `toJSON()`. This shape is **private** to `InMemoryUpdatesStore` - * — it is NOT part of the `UpdatesStore` interface; a file-backed or database - * store may persist however it likes. - * - * For backward compatibility the constructor also accepts a legacy flat - * `{ [signal]: { [uri]: number } }` object (no `updates`/`handled` keys), - * treating the whole object as the `updates` relation with empty handled state. - */ -export type SerializedUpdatesStore = { - updates: { [signal: string]: { [uri: string]: number } }; - handled: { [signal: string]: { [cell: string]: { [uri: string]: number } } }; -}; - -/** - * Yield ordering for read methods on `UpdatesStore`. Both options yield - * the same set of entries — only the order differs. - * - * - `"stamp"` (default) — last-transaction-id ascending. Best when the - * consumer wants temporal-order replay. - * - `"uri"` — URI ascending. Best when the consumer wants to collapse - * per-URI work in one forward pass, or to merge multiple read streams - * by URI without buffering (see `readCellUpdates`). - */ -export type ReadOrderBy = "stamp" | "uri"; - -/** - * Per-`(signal, uri)` updates log plus a per-cell handled dimension, used by - * handlers driving multi-stage dataflow pipelines. Sits alongside - * `TransactionStore` and `DataflowGraph` as a leaf of `@statewalker/shared-dataflow`. - * - * Two logical relations: - * - * - `updates(signal, uri) -> stamp` — written by `setUpdate`, read by - * `readEntries` / `readUpdates`. Upsert by `(signal, uri)`; last write wins; - * stamps are caller-supplied and never validated beyond finiteness. - * - `handled(signal, cell, uri) -> stamp` — written by `handleUpdate`, consulted - * (never yielded) by `readUpdates`. Records how far a cell has caught up to a - * signal's updates, independently per cell. - * - * The interface is storage-agnostic: it references no key encoding, separator, - * or serialization, so it maps 1:1 onto a relational store (two tables; - * `readUpdates` = indexed left-join; `removeUpdate` = `ON DELETE CASCADE`). - * - * Caller responsibilities not enforced by the store: stamp discipline, signal - * naming consistency with any `DataflowGraph` topology, tombstone naming - * conventions. - */ -export interface UpdatesStore { - /** - * Read updates on a signal whose stamp > `since`, optionally filtered by - * URI prefix. Yields full `UpdateEntry` objects. - * - * - `since` is exclusive: `stamp > since`. Use `since = 0` to read everything. - * - `uriPrefix` (optional, defaults to no filter) restricts to entries whose - * `uri.startsWith(uriPrefix)`. An empty string is treated as no filter. - * - `orderBy` (optional, default `"stamp"`) — see `ReadOrderBy`. - */ - readEntries(opts: { - signal: Signal; - since: number; - uriPrefix?: string; - orderBy?: ReadOrderBy; - }): AsyncIterable; - - /** - * Per-cell diff: yields the updates on `signal` that `cell` hasn't caught - * up to yet. For each URI present on `signal`, comparing its update stamp - * to that cell's handled stamp for the same URI: - * - * - update stamp > cell's handled stamp (or handled absent → treated as 0): - * yielded. - * - update stamp <= cell's handled stamp: NOT yielded — the cell is already - * caught up. - * - * URIs that exist only as handled state (no update row) are NEVER yielded. - * - * Default order is update-stamp-ascending. Pass `orderBy: "uri"` for - * URI-ascending order — used by `readCellUpdates` to merge several per-signal - * streams in O(1) buffering. `uriPrefix` filters the same way as `readEntries`. - * - * Caller contract: after handling a yielded entry, call `handleUpdate` with - * stamp >= the yielded update stamp to advance the per-URI watermark for this - * cell. Otherwise the URI appears again on the next call. - */ - readUpdates(opts: { - signal: Signal; - cell: string; - uriPrefix?: string; - orderBy?: ReadOrderBy; - }): AsyncIterable; - - /** Upsert an update by `(signal, uri)`. Replaces blindly — last write wins. */ - setUpdate(entry: UpdateEntry): Promise; - /** Sequential equivalent of `entries.forEach(setUpdate)`. No atomicity promise. */ - setUpdates(entries: ReadonlyArray): Promise; - - /** - * Record that `cell` has handled the update `(signal, uri)` at `stamp` - * (intended to be the upstream stamp the cell observed). Upsert by - * `(signal, cell, uri)`; replaces blindly. Never touches `updates` rows. - */ - handleUpdate(entry: HandledEntry): Promise; - /** Sequential equivalent of `entries.forEach(handleUpdate)`. No atomicity promise. */ - handleUpdates(entries: ReadonlyArray): Promise; - - /** - * Remove every handled row recorded by `cell` against `signal`, resetting - * that cell's per-URI watermark for the signal so all of the signal's - * updates re-appear in the cell's next `readUpdates`. Never touches - * `updates` rows or any other cell's handled rows. Returns the number of - * handled rows removed. Used by restart/reset flows. - */ - clearHandled(key: { signal: Signal; cell: string }): Promise; - - /** - * Remove the update identified by `(signal, uri)`, AND cascade-remove every - * cell's handled row for that same `(signal, uri)`. Used by tombstone- - * consuming handlers to clean up after a deletion has propagated; the cascade - * prevents a re-created URI from being masked by a stale handled stamp. - * No-op if the update does not exist. - */ - removeUpdate(key: { signal: Signal; uri: string }): Promise; - /** Sequential equivalent of `keys.forEach(removeUpdate)`. No atomicity promise. */ - removeUpdates( - keys: ReadonlyArray<{ signal: Signal; uri: string }>, - ): Promise; -} diff --git a/packages/shared-dataflow/tests/dataflow-graph.test.ts b/packages/shared-dataflow/tests/dataflow-graph.test.ts deleted file mode 100644 index 2b72f89..0000000 --- a/packages/shared-dataflow/tests/dataflow-graph.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DataflowGraph } from "../src/dataflow-graph.js"; -import type { CellDefinition, CellId } from "../src/types.js"; - -/** - * Helpers - */ -function indexOf(order: CellId[], id: CellId): number { - const i = order.indexOf(id); - if (i < 0) throw new Error(`Expected "${id}" in order [${order.join(", ")}]`); - return i; -} - -function expectBefore(order: CellId[], a: CellId, b: CellId): void { - expect(indexOf(order, a)).toBeLessThan(indexOf(order, b)); -} - -describe("DataflowGraph — construction", () => { - it("rejects duplicate cell ids", () => { - const cells: CellDefinition[] = [ - { id: "A", inputs: [], outputs: ["x"] }, - { id: "A", inputs: ["x"], outputs: [] }, - ]; - expect(() => new DataflowGraph(cells)).toThrow(/Duplicate cell id: A/); - }); - - it("indexes signal → producers and signal → consumers", () => { - const g = new DataflowGraph([ - { id: "A", inputs: [], outputs: ["x"] }, - { id: "B", inputs: [], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: ["y"] }, - { id: "D", inputs: ["y"], outputs: [] }, - ]); - - expect(g.getCellsProducing("x")).toEqual(new Set(["A", "B"])); - expect(g.getCellsProducing("y")).toEqual(new Set(["C"])); - expect(g.getCellsConsuming("x")).toEqual(new Set(["C"])); - expect(g.getCellsConsuming("y")).toEqual(new Set(["D"])); - expect(g.getCellsProducing("missing")).toEqual(new Set()); - }); - - it("returns input/output lists per cell", () => { - const g = new DataflowGraph([{ id: "A", inputs: ["x", "y"], outputs: ["z"] }]); - expect(g.getCellInputs("A")).toEqual(["x", "y"]); - expect(g.getCellOutputs("A")).toEqual(["z"]); - expect(g.getCellInputs("missing")).toEqual([]); - }); -}); - -describe("DataflowGraph — getExecutionOrder: trivial cases", () => { - it("returns [] when no signals are changed", () => { - const g = new DataflowGraph([ - { id: "A", inputs: [], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - expect(g.getExecutionOrder([])).toEqual([]); - }); - - it("returns [] when changed signal has no consumers", () => { - const g = new DataflowGraph([{ id: "A", inputs: [], outputs: ["x"] }]); - expect(g.getExecutionOrder(["unrelated"])).toEqual([]); - }); - - it("returns single seed when seed produces nothing", () => { - const g = new DataflowGraph([ - { id: "A", inputs: [], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: [] }, - ]); - expect(g.getExecutionOrder(["x"])).toEqual(["C"]); - }); -}); - -describe("DataflowGraph — forward propagation", () => { - it("walks transitively through consumers", () => { - const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["y"], outputs: [] }, - { id: "D", inputs: [], outputs: [] }, // unrelated, must not appear - ]); - const order = g.getExecutionOrder(["s"]); - expect(new Set(order)).toEqual(new Set(["A", "B", "C"])); - expectBefore(order, "A", "B"); - expectBefore(order, "B", "C"); - }); - - it("does not propagate upstream when only an output signal changes", () => { - // Even if x is produced by A, changing x externally only impacts consumers - // of x — A itself is not re-run. - const g = new DataflowGraph([ - { id: "A", inputs: [], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - expect(g.getExecutionOrder(["x"])).toEqual(["B"]); - }); -}); - -describe("DataflowGraph — barrier semantics with multiple producers", () => { - it("schedules a consumer after ALL impacted producers of its inputs", () => { - // A→x, B→x, C reads x. Triggering s only includes A (and C), - // so C must run after A — but B is not in this run. - const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: [], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: [] }, - ]); - - const order = g.getExecutionOrder(["s"]); - expect(new Set(order)).toEqual(new Set(["A", "C"])); - expectBefore(order, "A", "C"); - }); - - it("waits for both producers when both are impacted", () => { - // s impacts A and B (both produce x), C reads x. - const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["s"], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: [] }, - ]); - - const order = g.getExecutionOrder(["s"]); - expect(new Set(order)).toEqual(new Set(["A", "B", "C"])); - expectBefore(order, "A", "C"); - expectBefore(order, "B", "C"); - // A and B are independent — both orderings are valid. - }); - - it("ignores producers that are NOT in the impacted set when ordering", () => { - // A→x (impacted via s), B→x (NOT impacted), C reads x. - // C must wait for A but not for B. - const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["unrelated"], outputs: ["x"] }, - { id: "C", inputs: ["x"], outputs: [] }, - ]); - - const order = g.getExecutionOrder(["s"]); - expect(order).not.toContain("B"); - expectBefore(order, "A", "C"); - }); -}); - -describe("DataflowGraph — diamond and fan-out shapes", () => { - it("orders a classic diamond correctly", () => { - // A - // / \ - // B C - // \ / - // D - const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["x"], outputs: ["z"] }, - { id: "D", inputs: ["y", "z"], outputs: [] }, - ]); - - const order = g.getExecutionOrder(["s"]); - expect(new Set(order)).toEqual(new Set(["A", "B", "C", "D"])); - expectBefore(order, "A", "B"); - expectBefore(order, "A", "C"); - expectBefore(order, "B", "D"); - expectBefore(order, "C", "D"); - }); - - it("includes both branches of a fan-out from one signal change", () => { - const g = new DataflowGraph([ - { id: "ROOT", inputs: ["s"], outputs: ["x"] }, - { id: "L", inputs: ["x"], outputs: [] }, - { id: "R", inputs: ["x"], outputs: [] }, - ]); - const order = g.getExecutionOrder(["s"]); - expect(new Set(order)).toEqual(new Set(["ROOT", "L", "R"])); - expectBefore(order, "ROOT", "L"); - expectBefore(order, "ROOT", "R"); - }); -}); - -describe("DataflowGraph — multiple changed signals", () => { - it("merges seed sets without duplicating cells", () => { - const g = new DataflowGraph([ - { id: "A", inputs: ["s1", "s2"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const order = g.getExecutionOrder(["s1", "s2"]); - expect(order).toEqual(["A", "B"]); - }); - - it("seeds multiple disjoint consumers from different signals", () => { - const g = new DataflowGraph([ - { id: "A", inputs: ["s1"], outputs: [] }, - { id: "B", inputs: ["s2"], outputs: [] }, - { id: "C", inputs: ["s3"], outputs: [] }, - ]); - expect(new Set(g.getExecutionOrder(["s1", "s2"]))).toEqual(new Set(["A", "B"])); - }); -}); - -describe("DataflowGraph — cycle detection", () => { - it("throws if the impacted subgraph contains a cycle", () => { - // A reads x, writes y; B reads y, writes x — true cycle. - const g = new DataflowGraph([ - { id: "SEED", inputs: ["s"], outputs: ["x"] }, - { id: "A", inputs: ["x"], outputs: ["y"] }, - { id: "B", inputs: ["y"], outputs: ["x"] }, - ]); - expect(() => g.getExecutionOrder(["s"])).toThrow(/Cycle detected/); - }); - - it("reports only the cells actually in the cycle, not downstream consumers of it", () => { - // CYC_A and CYC_B form a cycle via signals x/y. - // DOWN reads x but never feeds back into the cycle — it is downstream - // of the cycle, not a member of it. The error must NOT list DOWN. - const g = new DataflowGraph([ - { id: "CYC_A", inputs: ["y"], outputs: ["x"] }, - { id: "CYC_B", inputs: ["x"], outputs: ["y"] }, - { id: "DOWN", inputs: ["x"], outputs: [] }, - ]); - let thrown: unknown; - try { - g.getExecutionOrder(["x"]); - } catch (e) { - thrown = e; - } - expect(thrown).toBeInstanceOf(Error); - const message = (thrown as Error).message; - expect(message).toMatch(/Cycle detected/); - expect(message).toContain("CYC_A"); - expect(message).toContain("CYC_B"); - expect(message).not.toContain("DOWN"); - }); - - it("does NOT throw if a cycle exists outside the impacted subgraph", () => { - // Cycle is on signals p/q, but the run only touches A→B. - const g = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - { id: "CYC1", inputs: ["p"], outputs: ["q"] }, - { id: "CYC2", inputs: ["q"], outputs: ["p"] }, - ]); - expect(() => g.getExecutionOrder(["s"])).not.toThrow(); - }); -}); - -describe("DataflowGraph — defensive copies", () => { - it("returns fresh sets / arrays so callers cannot mutate internals", () => { - const g = new DataflowGraph([{ id: "A", inputs: ["x"], outputs: ["y"] }]); - - const consumers = g.getCellsConsuming("x"); - consumers.add("HACKED"); - expect(g.getCellsConsuming("x")).toEqual(new Set(["A"])); - - const producers = g.getCellsProducing("y"); - producers.clear(); - expect(g.getCellsProducing("y")).toEqual(new Set(["A"])); - - const inputs = g.getCellInputs("A"); - inputs.push("HACKED"); - expect(g.getCellInputs("A")).toEqual(["x"]); - }); -}); diff --git a/packages/shared-dataflow/tests/in-memory-transaction-store.test.ts b/packages/shared-dataflow/tests/in-memory-transaction-store.test.ts deleted file mode 100644 index 9c69980..0000000 --- a/packages/shared-dataflow/tests/in-memory-transaction-store.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { InMemoryTransactionStore } from "../src/in-memory-transaction-store.js"; - -async function collect(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -describe("InMemoryTransactionStore — newTransactionId", () => { - it("returns strictly increasing values starting at 1", async () => { - const store = new InMemoryTransactionStore(); - expect(await store.newTransactionId()).toBe(1); - expect(await store.newTransactionId()).toBe(2); - expect(await store.newTransactionId()).toBe(3); - }); - - it("never repeats across many calls", async () => { - const store = new InMemoryTransactionStore(); - const ids = new Set(); - for (let i = 0; i < 1000; i++) ids.add(await store.newTransactionId()); - expect(ids.size).toBe(1000); - }); - - it("is independent across instances", async () => { - const a = new InMemoryTransactionStore(); - const b = new InMemoryTransactionStore(); - await a.newTransactionId(); - await a.newTransactionId(); - expect(await b.newTransactionId()).toBe(1); - }); -}); - -describe("InMemoryTransactionStore — getCellTransaction", () => { - it("returns 0 for an unknown cell", async () => { - const store = new InMemoryTransactionStore(); - expect(await store.getCellTransaction("unknown")).toBe(0); - }); - - it("returns the value last set", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 5); - expect(await store.getCellTransaction("A")).toBe(5); - }); - - it("setCellTransaction overwrites", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 5); - await store.setCellTransaction("A", 12); - expect(await store.getCellTransaction("A")).toBe(12); - }); - - it("isolates cells from each other", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 5); - await store.setCellTransaction("B", 7); - expect(await store.getCellTransaction("A")).toBe(5); - expect(await store.getCellTransaction("B")).toBe(7); - }); -}); - -describe("InMemoryTransactionStore — getCellsTransactions", () => { - it("yields nothing when store is empty", async () => { - const store = new InMemoryTransactionStore(); - expect(await collect(store.getCellsTransactions())).toEqual([]); - expect(await collect(store.getCellsTransactions(0))).toEqual([]); - }); - - it("yields all recorded cells when sinceTransactionId is omitted", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 1); - await store.setCellTransaction("B", 2); - await store.setCellTransaction("C", 3); - - const all = await collect(store.getCellsTransactions()); - expect(new Set(all)).toEqual( - new Set<[string, number]>([ - ["A", 1], - ["B", 2], - ["C", 3], - ]), - ); - }); - - it("filters strictly: yields only cells with tx > sinceTransactionId", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 1); - await store.setCellTransaction("B", 5); - await store.setCellTransaction("C", 5); - await store.setCellTransaction("D", 9); - - const since5 = await collect(store.getCellsTransactions(5)); - expect(new Set(since5)).toEqual(new Set<[string, number]>([["D", 9]])); - }); - - it("treats sinceTransactionId of 0 as 'all with strictly positive tx'", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 1); - await store.setCellTransaction("B", 2); - - const since0 = await collect(store.getCellsTransactions(0)); - expect(new Set(since0)).toEqual( - new Set<[string, number]>([ - ["A", 1], - ["B", 2], - ]), - ); - }); -}); - -describe("InMemoryTransactionStore — removeCellTransactions", () => { - it("removes a recorded cell so getCellTransaction returns 0", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 7); - await store.removeCellTransactions("A"); - expect(await store.getCellTransaction("A")).toBe(0); - }); - - it("excludes the removed cell from getCellsTransactions iteration", async () => { - const store = new InMemoryTransactionStore(); - await store.setCellTransaction("A", 1); - await store.setCellTransaction("B", 2); - await store.removeCellTransactions("A"); - - const all = await collect(store.getCellsTransactions()); - expect(new Set(all)).toEqual(new Set<[string, number]>([["B", 2]])); - }); - - it("is a no-op for an unknown cell", async () => { - const store = new InMemoryTransactionStore(); - await expect(store.removeCellTransactions("missing")).resolves.toBeUndefined(); - }); -}); diff --git a/packages/shared-dataflow/tests/in-memory-updates-store.test.ts b/packages/shared-dataflow/tests/in-memory-updates-store.test.ts deleted file mode 100644 index d97a592..0000000 --- a/packages/shared-dataflow/tests/in-memory-updates-store.test.ts +++ /dev/null @@ -1,629 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { InMemoryUpdatesStore } from "../src/in-memory-updates-store.js"; -import type { SerializedUpdatesStore, UpdateEntry } from "../src/updates-store.js"; - -async function collect(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -describe("InMemoryUpdatesStore — round-trip & shape", () => { - it("yields a single saved entry back via readEntries", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "files", uri: "f1", stamp: 5 }); - const got = await collect(store.readEntries({ signal: "files", since: 0 })); - expect(got).toEqual([{ signal: "files", uri: "f1", stamp: 5 }]); - }); - - it("yielded entries have exactly { signal, uri, stamp } and the queried signal", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "files", uri: "f1", stamp: 1 }); - const got = await collect(store.readEntries({ signal: "files", since: 0 })); - expect(got).toHaveLength(1); - const first = got[0]; - if (!first) throw new Error("unreachable — length asserted above"); - expect(Object.keys(first).sort()).toEqual(["signal", "stamp", "uri"]); - expect(first.signal).toBe("files"); - }); -}); - -describe("InMemoryUpdatesStore — readEntries filters", () => { - it("`since` is exclusive — entries with stamp == since are not yielded", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 5 }, - { signal: "x", uri: "b", stamp: 7 }, - ]); - const got = await collect(store.readEntries({ signal: "x", since: 5 })); - expect(got).toEqual([{ signal: "x", uri: "b", stamp: 7 }]); - }); - - it("`since = 0` yields all entries", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 2 }, - { signal: "x", uri: "c", stamp: 3 }, - ]); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got.map((e) => e.stamp)).toEqual([1, 2, 3]); - }); - - it("`since` larger than any stamp yields nothing", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 10 }, - ]); - const got = await collect(store.readEntries({ signal: "x", since: 1000 })); - expect(got).toEqual([]); - }); - - it("`uriPrefix` selects only entries whose uri starts with the prefix", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "files", uri: "/a/1", stamp: 1 }, - { signal: "files", uri: "/a/2", stamp: 2 }, - { signal: "files", uri: "/b/1", stamp: 3 }, - ]); - const got = await collect(store.readEntries({ signal: "files", since: 0, uriPrefix: "/a/" })); - expect(got.map((e) => e.uri).sort()).toEqual(["/a/1", "/a/2"]); - }); - - it("empty `uriPrefix` is equivalent to no filter", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 2 }, - ]); - const withEmpty = await collect(store.readEntries({ signal: "x", since: 0, uriPrefix: "" })); - const without = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(withEmpty).toEqual(without); - }); -}); - -describe("InMemoryUpdatesStore — readEntries ordering", () => { - it("yields entries in stamp-ascending order regardless of save order", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "c", stamp: 9 }); - await store.setUpdate({ signal: "x", uri: "a", stamp: 3 }); - await store.setUpdate({ signal: "x", uri: "b", stamp: 5 }); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got.map((e) => e.stamp)).toEqual([3, 5, 9]); - }); -}); - -describe("InMemoryUpdatesStore — empty / unknown queries", () => { - it("yields nothing for an unknown signal on an empty store, without throwing", async () => { - const store = new InMemoryUpdatesStore(); - const got = await collect(store.readEntries({ signal: "never-seen", since: 0 })); - expect(got).toEqual([]); - }); - - it("yields nothing for a known signal whose entries are all <= since", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 2 }, - ]); - const got = await collect(store.readEntries({ signal: "x", since: 2 })); - expect(got).toEqual([]); - }); -}); - -describe("InMemoryUpdatesStore — upsert blind replace", () => { - it("second save with greater stamp wins", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "a", stamp: 3 }); - await store.setUpdate({ signal: "x", uri: "a", stamp: 7 }); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got).toEqual([{ signal: "x", uri: "a", stamp: 7 }]); - }); - - it("second save with smaller stamp also wins (blind replace, no monotonicity)", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "a", stamp: 7 }); - await store.setUpdate({ signal: "x", uri: "a", stamp: 3 }); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got).toEqual([{ signal: "x", uri: "a", stamp: 3 }]); - }); - - it("saves on different signals do not interfere", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "a", stamp: 1 }); - await store.setUpdate({ signal: "y", uri: "a", stamp: 2 }); - expect(await collect(store.readEntries({ signal: "x", since: 0 }))).toEqual([ - { signal: "x", uri: "a", stamp: 1 }, - ]); - expect(await collect(store.readEntries({ signal: "y", since: 0 }))).toEqual([ - { signal: "y", uri: "a", stamp: 2 }, - ]); - }); - - it("saves on different URIs of the same signal do not interfere", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "a", stamp: 1 }); - await store.setUpdate({ signal: "x", uri: "b", stamp: 2 }); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got.map((e) => [e.uri, e.stamp])).toEqual([ - ["a", 1], - ["b", 2], - ]); - }); -}); - -describe("InMemoryUpdatesStore — deletion (with cascade)", () => { - it("removeUpdate removes an existing row", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "a", stamp: 5 }); - await store.removeUpdate({ signal: "x", uri: "a" }); - expect(await collect(store.readEntries({ signal: "x", since: 0 }))).toEqual([]); - }); - - it("removeUpdate on a non-existent key is a no-op (does not throw)", async () => { - const store = new InMemoryUpdatesStore(); - await expect(store.removeUpdate({ signal: "x", uri: "never" })).resolves.toBeUndefined(); - await expect( - store.removeUpdate({ signal: "never-seen", uri: "anything" }), - ).resolves.toBeUndefined(); - }); - - it("removeUpdate is local — does not affect other update keys", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 2 }, - { signal: "y", uri: "a", stamp: 3 }, - ]); - await store.removeUpdate({ signal: "x", uri: "a" }); - expect(await collect(store.readEntries({ signal: "x", since: 0 }))).toEqual([ - { signal: "x", uri: "b", stamp: 2 }, - ]); - expect(await collect(store.readEntries({ signal: "y", since: 0 }))).toEqual([ - { signal: "y", uri: "a", stamp: 3 }, - ]); - }); - - it("removeUpdate cascades: clears every cell's handled row for (signal, uri)", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "s", uri: "u", stamp: 5 }); - await store.handleUpdate({ signal: "s", uri: "u", cell: "c1", stamp: 5 }); - await store.handleUpdate({ signal: "s", uri: "u", cell: "c2", stamp: 5 }); - - await store.removeUpdate({ signal: "s", uri: "u" }); - await store.setUpdate({ signal: "s", uri: "u", stamp: 9 }); - - expect(await collect(store.readUpdates({ signal: "s", cell: "c1" }))).toEqual([ - { signal: "s", uri: "u", stamp: 9 }, - ]); - expect(await collect(store.readUpdates({ signal: "s", cell: "c2" }))).toEqual([ - { signal: "s", uri: "u", stamp: 9 }, - ]); - }); - - it("cascade does not touch other URIs' handled rows", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "s", uri: "u1", stamp: 5 }, - { signal: "s", uri: "u2", stamp: 5 }, - ]); - await store.handleUpdate({ signal: "s", uri: "u1", cell: "c", stamp: 5 }); - await store.handleUpdate({ signal: "s", uri: "u2", cell: "c", stamp: 5 }); - - await store.removeUpdate({ signal: "s", uri: "u1" }); - - // u2 still handled by c → not yielded. - expect(await collect(store.readUpdates({ signal: "s", cell: "c" }))).toEqual([]); - }); -}); - -describe("InMemoryUpdatesStore — batch operations", () => { - it("setUpdates applies every entry", async () => { - const store = new InMemoryUpdatesStore(); - const entries: UpdateEntry[] = [ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 2 }, - ]; - await store.setUpdates(entries); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got).toEqual(entries); - }); - - it("removeUpdates removes every listed key", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "a", stamp: 1 }, - { signal: "x", uri: "b", stamp: 2 }, - { signal: "x", uri: "c", stamp: 3 }, - ]); - await store.removeUpdates([ - { signal: "x", uri: "a" }, - { signal: "x", uri: "b" }, - ]); - expect(await collect(store.readEntries({ signal: "x", since: 0 }))).toEqual([ - { signal: "x", uri: "c", stamp: 3 }, - ]); - }); - - it("handleUpdates marks every (signal, uri, cell) handled", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "s", uri: "a", stamp: 1 }, - { signal: "s", uri: "b", stamp: 1 }, - ]); - await store.handleUpdates([ - { signal: "s", uri: "a", cell: "c", stamp: 1 }, - { signal: "s", uri: "b", cell: "c", stamp: 1 }, - ]); - expect(await collect(store.readUpdates({ signal: "s", cell: "c" }))).toEqual([]); - }); -}); - -describe("InMemoryUpdatesStore — handleUpdate / per-cell isolation", () => { - it("two cells handle the same (signal, uri) independently", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "file-source", uri: "/a.pdf", stamp: 8 }); - - await store.handleUpdate({ signal: "file-source", uri: "/a.pdf", cell: "extract", stamp: 8 }); - - expect(await collect(store.readUpdates({ signal: "file-source", cell: "extract" }))).toEqual( - [], - ); - expect(await collect(store.readUpdates({ signal: "file-source", cell: "preview" }))).toEqual([ - { signal: "file-source", uri: "/a.pdf", stamp: 8 }, - ]); - // The raw update row is untouched. - expect(await collect(store.readEntries({ signal: "file-source", since: 0 }))).toEqual([ - { signal: "file-source", uri: "/a.pdf", stamp: 8 }, - ]); - }); - - it("recording handled state never alters update rows / does not leak into readEntries", async () => { - const store = new InMemoryUpdatesStore(); - await store.handleUpdate({ signal: "s", uri: "u", cell: "c", stamp: 1 }); - expect(await collect(store.readEntries({ signal: "s", since: 0 }))).toEqual([]); - }); - - it("accepts arbitrary signal/cell strings (spaces, delimiters) and round-trips them", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "a b", uri: "u", stamp: 1 }); - await store.handleUpdate({ signal: "a b", uri: "u", cell: "x y", stamp: 1 }); - expect(await collect(store.readUpdates({ signal: "a b", cell: "x y" }))).toEqual([]); - expect(await collect(store.readUpdates({ signal: "a b", cell: "other" }))).toEqual([ - { signal: "a b", uri: "u", stamp: 1 }, - ]); - }); - - it("rejects a non-finite handled stamp", async () => { - const store = new InMemoryUpdatesStore(); - await expect( - store.handleUpdate({ signal: "s", uri: "u", cell: "c", stamp: Number.NaN }), - ).rejects.toThrow(/finite|NaN|stamp/i); - }); -}); - -describe("InMemoryUpdatesStore — clearHandled (per-cell watermark reset)", () => { - it("clears one cell's handled rows for a signal; the cell re-sees all updates", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "s", uri: "a", stamp: 5 }, - { signal: "s", uri: "b", stamp: 6 }, - ]); - await store.handleUpdate({ signal: "s", uri: "a", cell: "c", stamp: 5 }); - await store.handleUpdate({ signal: "s", uri: "b", cell: "c", stamp: 6 }); - expect(await collect(store.readUpdates({ signal: "s", cell: "c" }))).toEqual([]); - - const removed = await store.clearHandled({ signal: "s", cell: "c" }); - expect(removed).toBe(2); - expect(await collect(store.readUpdates({ signal: "s", cell: "c" }))).toEqual([ - { signal: "s", uri: "a", stamp: 5 }, - { signal: "s", uri: "b", stamp: 6 }, - ]); - }); - - it("does not touch update rows or other cells' handled state", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "s", uri: "a", stamp: 5 }); - await store.handleUpdate({ signal: "s", uri: "a", cell: "c1", stamp: 5 }); - await store.handleUpdate({ signal: "s", uri: "a", cell: "c2", stamp: 5 }); - - await store.clearHandled({ signal: "s", cell: "c1" }); - - // Update row intact. - expect(await collect(store.readEntries({ signal: "s", since: 0 }))).toEqual([ - { signal: "s", uri: "a", stamp: 5 }, - ]); - // c1 reset, c2 untouched. - expect(await collect(store.readUpdates({ signal: "s", cell: "c1" }))).toEqual([ - { signal: "s", uri: "a", stamp: 5 }, - ]); - expect(await collect(store.readUpdates({ signal: "s", cell: "c2" }))).toEqual([]); - }); - - it("returns 0 when the cell has no handled rows for the signal", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "s", uri: "a", stamp: 5 }); - expect(await store.clearHandled({ signal: "s", cell: "never" })).toBe(0); - expect(await store.clearHandled({ signal: "missing", cell: "c" })).toBe(0); - }); -}); - -describe("InMemoryUpdatesStore — readUpdates (per-cell diff)", () => { - it("yields every update when the cell has handled nothing", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src", uri: "a", stamp: 1 }, - { signal: "src", uri: "b", stamp: 2 }, - ]); - const got = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(got).toEqual([ - { signal: "src", uri: "a", stamp: 1 }, - { signal: "src", uri: "b", stamp: 2 }, - ]); - }); - - it("skips URIs where the cell's handled stamp >= update stamp", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src", uri: "a", stamp: 5 }, - { signal: "src", uri: "b", stamp: 7 }, - ]); - await store.handleUpdate({ signal: "src", uri: "a", cell: "ext", stamp: 5 }); // caught up - await store.handleUpdate({ signal: "src", uri: "b", cell: "ext", stamp: 6 }); // lagging - const got = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(got).toEqual([{ signal: "src", uri: "b", stamp: 7 }]); - }); - - it("URI present only as handled state (no update row) is not yielded", async () => { - const store = new InMemoryUpdatesStore(); - await store.handleUpdate({ signal: "src", uri: "a", cell: "ext", stamp: 3 }); - const got = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(got).toEqual([]); - }); - - it("yields entries in update-stamp-ascending order by default", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src", uri: "c", stamp: 9 }, - { signal: "src", uri: "a", stamp: 3 }, - { signal: "src", uri: "b", stamp: 5 }, - ]); - const got = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(got.map((e) => e.stamp)).toEqual([3, 5, 9]); - }); - - it("orderBy: 'uri' yields URI-ascending", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src", uri: "z", stamp: 1 }, - { signal: "src", uri: "a", stamp: 9 }, - { signal: "src", uri: "m", stamp: 5 }, - ]); - const got = await collect(store.readUpdates({ signal: "src", cell: "ext", orderBy: "uri" })); - expect(got.map((e) => e.uri)).toEqual(["a", "m", "z"]); - }); - - it("uriPrefix restricts to URIs whose path matches the prefix", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src", uri: "/a/1", stamp: 1 }, - { signal: "src", uri: "/a/2", stamp: 2 }, - { signal: "src", uri: "/b/1", stamp: 3 }, - ]); - const got = await collect(store.readUpdates({ signal: "src", cell: "ext", uriPrefix: "/a/" })); - expect(got.map((e) => e.uri).sort()).toEqual(["/a/1", "/a/2"]); - }); - - it("yields nothing when the signal has no update rows", async () => { - const store = new InMemoryUpdatesStore(); - await store.handleUpdate({ signal: "ext", uri: "a", cell: "c", stamp: 1 }); - const got = await collect(store.readUpdates({ signal: "never-emitted", cell: "c" })); - expect(got).toEqual([]); - }); - - it("after the cell handles a yielded update, the URI no longer appears", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "src", uri: "a", stamp: 5 }); - - const first = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(first).toEqual([{ signal: "src", uri: "a", stamp: 5 }]); - - await store.handleUpdate({ signal: "src", uri: "a", cell: "ext", stamp: 5 }); - - const second = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(second).toEqual([]); - }); - - it("a newer update after handling reappears", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "src", uri: "a", stamp: 5 }); - await store.handleUpdate({ signal: "src", uri: "a", cell: "ext", stamp: 5 }); - expect(await collect(store.readUpdates({ signal: "src", cell: "ext" }))).toEqual([]); - - await store.setUpdate({ signal: "src", uri: "a", stamp: 9 }); - expect(await collect(store.readUpdates({ signal: "src", cell: "ext" }))).toEqual([ - { signal: "src", uri: "a", stamp: 9 }, - ]); - }); - - it("empty uriPrefix is equivalent to no prefix", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src", uri: "a", stamp: 1 }, - { signal: "src", uri: "b", stamp: 2 }, - ]); - const withEmpty = await collect( - store.readUpdates({ signal: "src", cell: "ext", uriPrefix: "" }), - ); - const without = await collect(store.readUpdates({ signal: "src", cell: "ext" })); - expect(withEmpty).toEqual(without); - }); -}); - -describe("InMemoryUpdatesStore — JSON round-trip (two relations)", () => { - it("new InMemoryUpdatesStore(prev.snapshot()) preserves updates and handled state", async () => { - const prev = new InMemoryUpdatesStore(); - await prev.setUpdates([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - { signal: "content", uri: "f1", stamp: 3 }, - ]); - await prev.handleUpdate({ signal: "files", uri: "f1", cell: "c", stamp: 1 }); - - const next = new InMemoryUpdatesStore(prev.snapshot()); - - expect(await collect(next.readEntries({ signal: "files", since: 0 }))).toEqual( - await collect(prev.readEntries({ signal: "files", since: 0 })), - ); - // handled state survived: f1 handled by c, f2 not. - expect(await collect(next.readUpdates({ signal: "files", cell: "c" }))).toEqual([ - { signal: "files", uri: "f2", stamp: 2 }, - ]); - expect(await collect(next.readUpdates({ signal: "files", cell: "other" }))).toEqual([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - }); - - it("JSON.parse(JSON.stringify(store)) round-trips updates and handled through the constructor", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - await store.handleUpdate({ signal: "files", uri: "f1", cell: "c", stamp: 1 }); - const blob = JSON.stringify(store); - const restored = new InMemoryUpdatesStore(JSON.parse(blob)); - expect(await collect(restored.readEntries({ signal: "files", since: 0 }))).toEqual([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - expect(await collect(restored.readUpdates({ signal: "files", cell: "c" }))).toEqual([ - { signal: "files", uri: "f2", stamp: 2 }, - ]); - }); - - it("constructor accepts a legacy flat snapshot (updates-only) as updates", async () => { - const legacy = { files: { f1: 1, f2: 2 } } as unknown as SerializedUpdatesStore; - const store = new InMemoryUpdatesStore(legacy); - expect(await collect(store.readEntries({ signal: "files", since: 0 }))).toEqual([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - }); - - it("constructor defensively copies initialState — mutating input does not affect store", async () => { - const state: SerializedUpdatesStore = { - updates: { files: { f1: 1, f2: 2 } }, - handled: {}, - }; - const store = new InMemoryUpdatesStore(state); - const filesIn = state.updates.files; - if (!filesIn) throw new Error("unreachable — set above"); - delete filesIn.f1; - filesIn.f2 = 999; - const got = await collect(store.readEntries({ signal: "files", since: 0 })); - expect(got).toEqual([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - }); - - it("snapshot() returns a fresh object — mutating it does not affect store", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - const snap = store.snapshot(); - const filesOut = snap.updates.files; - if (!filesOut) throw new Error("unreachable — files just saved"); - delete filesOut.f1; - filesOut.f2 = 999; - const got = await collect(store.readEntries({ signal: "files", since: 0 })); - expect(got).toEqual([ - { signal: "files", uri: "f1", stamp: 1 }, - { signal: "files", uri: "f2", stamp: 2 }, - ]); - }); - - it("snapshot() of an empty store has empty updates and handled", async () => { - const store = new InMemoryUpdatesStore(); - expect(store.snapshot()).toEqual({ updates: {}, handled: {} }); - }); - - it("removeUpdate-emptied signal is absent from snapshot()", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "x", uri: "a", stamp: 1 }); - await store.removeUpdate({ signal: "x", uri: "a" }); - expect(store.snapshot()).toEqual({ updates: {}, handled: {} }); - }); - - it("preserves entries whose signal or uri is '__proto__' through serialization", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "__proto__", uri: "f1", stamp: 5 }); - await store.setUpdate({ signal: "files", uri: "__proto__", stamp: 9 }); - - const snap = store.snapshot(); - const proto = snap.updates.__proto__; - if (!proto) throw new Error("expected __proto__ signal in snapshot"); - expect(proto.f1).toBe(5); - - const files = snap.updates.files; - if (!files) throw new Error("expected files signal in snapshot"); - expect(files.__proto__).toBe(9); - - const blob = JSON.stringify(store); - expect(blob).toContain("__proto__"); - const restored = new InMemoryUpdatesStore(JSON.parse(blob)); - expect(await collect(restored.readEntries({ signal: "__proto__", since: 0 }))).toEqual([ - { signal: "__proto__", uri: "f1", stamp: 5 }, - ]); - expect(await collect(restored.readEntries({ signal: "files", since: 0 }))).toEqual([ - { signal: "files", uri: "__proto__", stamp: 9 }, - ]); - }); -}); - -describe("InMemoryUpdatesStore — orderBy 'stamp' vs 'uri'", () => { - it("readEntries defaults to stamp-ascending order", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "c", stamp: 9 }, - { signal: "x", uri: "a", stamp: 3 }, - { signal: "x", uri: "b", stamp: 5 }, - ]); - const got = await collect(store.readEntries({ signal: "x", since: 0 })); - expect(got.map((e) => e.uri)).toEqual(["a", "b", "c"]); - expect(got.map((e) => e.stamp)).toEqual([3, 5, 9]); - }); - - it("readEntries with orderBy: 'uri' yields URI-ascending regardless of stamp", async () => { - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "x", uri: "b", stamp: 9 }, - { signal: "x", uri: "a", stamp: 100 }, - { signal: "x", uri: "c", stamp: 1 }, - ]); - const got = await collect(store.readEntries({ signal: "x", since: 0, orderBy: "uri" })); - expect(got.map((e) => e.uri)).toEqual(["a", "b", "c"]); - expect(got.map((e) => e.stamp)).toEqual([100, 9, 1]); - }); -}); - -describe("InMemoryUpdatesStore — stamp validation", () => { - it("rejects a NaN stamp instead of storing an unreadable entry", async () => { - const store = new InMemoryUpdatesStore(); - await expect(store.setUpdate({ signal: "x", uri: "a", stamp: Number.NaN })).rejects.toThrow( - /finite|NaN|stamp/i, - ); - }); - - it("rejects a non-finite stamp (Infinity) for the same reason", async () => { - const store = new InMemoryUpdatesStore(); - await expect( - store.setUpdate({ signal: "x", uri: "a", stamp: Number.POSITIVE_INFINITY }), - ).rejects.toThrow(/finite|stamp/i); - }); -}); diff --git a/packages/shared-dataflow/tests/read-cell-updates.test.ts b/packages/shared-dataflow/tests/read-cell-updates.test.ts deleted file mode 100644 index d5f9a29..0000000 --- a/packages/shared-dataflow/tests/read-cell-updates.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DataflowGraph } from "../src/dataflow-graph.js"; -import { InMemoryUpdatesStore } from "../src/in-memory-updates-store.js"; -import { aggregateByUri, readCellUpdates } from "../src/read-cell-updates.js"; -import type { UpdateEntry } from "../src/updates-store.js"; - -async function collect(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -describe("readCellUpdates — graph-driven per-cell unhandled diff", () => { - it("yields the updates on a single upstream signal the cell hasn't handled", async () => { - const graph = new DataflowGraph([ - { id: "Extractor", inputs: ["sources"], outputs: ["extracted"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "sources", uri: "a", stamp: 1 }, - { signal: "sources", uri: "b", stamp: 3 }, - ]); - await store.handleUpdate({ signal: "sources", uri: "a", cell: "Extractor", stamp: 1 }); - const got = await collect(readCellUpdates(store, graph, "Extractor")); - expect(got).toEqual([{ signal: "sources", uri: "b", stamp: 3 }]); - }); - - it("yields entries from every upstream signal of the cell", async () => { - const graph = new DataflowGraph([ - { - id: "Pruner", - inputs: ["sources:removed", "meta:removed-topics"], - outputs: ["pruned"], - }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "sources:removed", uri: "a", stamp: 5 }, - { signal: "meta:removed-topics", uri: "a#topicX", stamp: 7 }, - ]); - const got = await collect(readCellUpdates(store, graph, "Pruner")); - expect(got).toEqual([ - { signal: "sources:removed", uri: "a", stamp: 5 }, - { signal: "meta:removed-topics", uri: "a#topicX", stamp: 7 }, - ]); - }); - - it("yields the same URI multiple times when it's fresh in multiple upstream signals", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-a", "src-b"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src-a", uri: "shared", stamp: 1 }, - { signal: "src-b", uri: "shared", stamp: 2 }, - ]); - const got = await collect(readCellUpdates(store, graph, "FanIn")); - expect(got).toEqual([ - { signal: "src-a", uri: "shared", stamp: 1 }, - { signal: "src-b", uri: "shared", stamp: 2 }, - ]); - }); - - it("respects per-cell watermark — URIs the cell handled on each input are excluded", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-a", "src-b"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src-a", uri: "x", stamp: 5 }, - { signal: "src-b", uri: "x", stamp: 7 }, - ]); - await store.handleUpdate({ signal: "src-a", uri: "x", cell: "FanIn", stamp: 5 }); - await store.handleUpdate({ signal: "src-b", uri: "x", cell: "FanIn", stamp: 7 }); - const got = await collect(readCellUpdates(store, graph, "FanIn")); - expect(got).toEqual([]); - }); - - it("yields nothing for a prober cell (no inputs)", async () => { - const graph = new DataflowGraph([{ id: "Scanner", inputs: [], outputs: ["sources"] }]); - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "sources", uri: "a", stamp: 1 }); - const got = await collect(readCellUpdates(store, graph, "Scanner")); - expect(got).toEqual([]); - }); - - it("works for a sink cell (inputs, no outputs) — yields and does NOT throw", async () => { - const graph = new DataflowGraph([{ id: "Index", inputs: ["content"], outputs: [] }]); - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "content", uri: "u", stamp: 2 }); - const got = await collect(readCellUpdates(store, graph, "Index")); - expect(got).toEqual([{ signal: "content", uri: "u", stamp: 2 }]); - }); - - it("two cells consuming the same input track handled state independently", async () => { - const graph = new DataflowGraph([ - { id: "Extract", inputs: ["file-source"], outputs: ["content"] }, - { id: "Preview", inputs: ["file-source"], outputs: [] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "file-source", uri: "/a.pdf", stamp: 8 }); - await store.handleUpdate({ signal: "file-source", uri: "/a.pdf", cell: "Extract", stamp: 8 }); - - expect(await collect(readCellUpdates(store, graph, "Extract"))).toEqual([]); - expect(await collect(readCellUpdates(store, graph, "Preview"))).toEqual([ - { signal: "file-source", uri: "/a.pdf", stamp: 8 }, - ]); - }); - - it("uriPrefix filters every upstream signal consistently", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-a", "src-b"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src-a", uri: "/keep/1", stamp: 1 }, - { signal: "src-a", uri: "/drop/1", stamp: 2 }, - { signal: "src-b", uri: "/keep/1", stamp: 3 }, - { signal: "src-b", uri: "/drop/2", stamp: 4 }, - ]); - const got = await collect(readCellUpdates(store, graph, "FanIn", { uriPrefix: "/keep/" })); - expect(got.map((e) => [e.signal, e.uri])).toEqual([ - ["src-a", "/keep/1"], - ["src-b", "/keep/1"], - ]); - }); - - it("merges entries from all upstream signals into a single URI-sorted stream", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-a", "src-b", "src-c"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src-a", uri: "z", stamp: 1 }, - { signal: "src-a", uri: "m", stamp: 9 }, - { signal: "src-b", uri: "a", stamp: 2 }, - { signal: "src-b", uri: "z", stamp: 3 }, - { signal: "src-c", uri: "m", stamp: 4 }, - ]); - const got = await collect(readCellUpdates(store, graph, "FanIn")); - expect(got.map((e) => [e.uri, e.signal])).toEqual([ - ["a", "src-b"], - ["m", "src-a"], - ["m", "src-c"], - ["z", "src-a"], - ["z", "src-b"], - ]); - }); - - it("breaks URI ties by signal-declaration order (graph.getCellInputs)", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-z", "src-a"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src-a", uri: "u", stamp: 1 }, - { signal: "src-z", uri: "u", stamp: 2 }, - ]); - const got = await collect(readCellUpdates(store, graph, "FanIn")); - expect(got.map((e) => e.signal)).toEqual(["src-z", "src-a"]); - }); - - it("yields nothing for an unknown cell id", async () => { - const graph = new DataflowGraph([ - { id: "Extractor", inputs: ["sources"], outputs: ["extracted"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdate({ signal: "sources", uri: "a", stamp: 1 }); - const got = await collect(readCellUpdates(store, graph, "Unknown")); - expect(got).toEqual([]); - }); - - it("an upstream signal with no entries contributes nothing but the next upstream still flows", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["empty-src", "live-src"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "live-src", uri: "a", stamp: 1 }, - { signal: "live-src", uri: "b", stamp: 2 }, - ]); - const got = await collect(readCellUpdates(store, graph, "FanIn")); - expect(got).toEqual([ - { signal: "live-src", uri: "a", stamp: 1 }, - { signal: "live-src", uri: "b", stamp: 2 }, - ]); - }); - - it("incrementally handling URIs monotonically shrinks the diff", async () => { - const graph = new DataflowGraph([ - { id: "Extractor", inputs: ["sources"], outputs: ["extracted"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "sources", uri: "a", stamp: 1 }, - { signal: "sources", uri: "b", stamp: 2 }, - { signal: "sources", uri: "c", stamp: 3 }, - ]); - - expect((await collect(readCellUpdates(store, graph, "Extractor"))).map((e) => e.uri)).toEqual([ - "a", - "b", - "c", - ]); - - await store.handleUpdate({ signal: "sources", uri: "a", cell: "Extractor", stamp: 1 }); - expect((await collect(readCellUpdates(store, graph, "Extractor"))).map((e) => e.uri)).toEqual([ - "b", - "c", - ]); - - await store.handleUpdate({ signal: "sources", uri: "b", cell: "Extractor", stamp: 2 }); - expect((await collect(readCellUpdates(store, graph, "Extractor"))).map((e) => e.uri)).toEqual([ - "c", - ]); - - await store.handleUpdate({ signal: "sources", uri: "c", cell: "Extractor", stamp: 3 }); - expect(await collect(readCellUpdates(store, graph, "Extractor"))).toEqual([]); - }); - - it("does not pre-buffer: the consumer can break early without draining all upstream entries", async () => { - let reads = 0; - const inner = new InMemoryUpdatesStore(); - await inner.setUpdates([ - ...Array.from({ length: 10 }, (_, i) => ({ - signal: "src-a", - uri: `a-${String(i).padStart(2, "0")}`, - stamp: i + 1, - })), - ...Array.from({ length: 10 }, (_, i) => ({ - signal: "src-b", - uri: `b-${String(i).padStart(2, "0")}`, - stamp: i + 1, - })), - ]); - - const spyStore = new Proxy(inner, { - get(target, prop, receiver) { - if (prop === "readUpdates") { - return async function* ( - this: unknown, - ...args: Parameters - ): AsyncIterable { - for await (const e of inner.readUpdates(...args)) { - reads += 1; - yield e; - } - }; - } - return Reflect.get(target, prop, receiver); - }, - }); - - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-a", "src-b"], outputs: ["fan-out"] }, - ]); - - let firstUri: string | undefined; - for await (const entry of readCellUpdates(spyStore, graph, "FanIn")) { - firstUri = entry.uri; - break; - } - expect(firstUri).toBe("a-00"); - expect(reads).toBeLessThan(20); - }); - - it("scales: 100 URIs across 3 upstream signals merge into a single URI-sorted stream", async () => { - const graph = new DataflowGraph([ - { id: "Big", inputs: ["src-a", "src-b", "src-c"], outputs: ["consolidated"] }, - ]); - const store = new InMemoryUpdatesStore(); - const expected: string[] = []; - for (let i = 0; i < 100; i++) { - const uri = `u-${String(i).padStart(3, "0")}`; - expected.push(uri); - const signal = ["src-a", "src-b", "src-c"][i % 3] as string; - await store.setUpdate({ signal, uri, stamp: ((i * 17) % 97) + 1 }); - } - const got = await collect(readCellUpdates(store, graph, "Big")); - expect(got).toHaveLength(100); - expect(got.map((e) => e.uri)).toEqual(expected); - }); -}); - -describe("aggregateByUri — collapse multi-upstream entries per URI", () => { - it("returns a Map keyed by URI with all contributing entries", async () => { - const entries: UpdateEntry[] = [ - { signal: "src-a", uri: "x", stamp: 1 }, - { signal: "src-b", uri: "x", stamp: 2 }, - { signal: "src-a", uri: "y", stamp: 3 }, - ]; - async function* gen(): AsyncIterable { - for (const e of entries) yield e; - } - const got = await aggregateByUri(gen()); - expect(got.size).toBe(2); - expect(got.get("x")).toEqual([ - { signal: "src-a", uri: "x", stamp: 1 }, - { signal: "src-b", uri: "x", stamp: 2 }, - ]); - expect(got.get("y")).toEqual([{ signal: "src-a", uri: "y", stamp: 3 }]); - }); - - it("preserves source insertion order in the Map", async () => { - const entries: UpdateEntry[] = [ - { signal: "s", uri: "b", stamp: 1 }, - { signal: "s", uri: "a", stamp: 2 }, - { signal: "s", uri: "c", stamp: 3 }, - ]; - async function* gen(): AsyncIterable { - for (const e of entries) yield e; - } - const got = await aggregateByUri(gen()); - expect([...got.keys()]).toEqual(["b", "a", "c"]); - }); - - it("returns an empty Map for an empty source", async () => { - async function* gen(): AsyncIterable { - // no yields - } - const got = await aggregateByUri(gen()); - expect(got.size).toBe(0); - }); - - it("composes with readCellUpdates to give one record per URI across upstream signals", async () => { - const graph = new DataflowGraph([ - { id: "FanIn", inputs: ["src-a", "src-b"], outputs: ["fan-out"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "src-a", uri: "x", stamp: 1 }, - { signal: "src-b", uri: "x", stamp: 2 }, - { signal: "src-a", uri: "y", stamp: 3 }, - ]); - const got = await aggregateByUri(readCellUpdates(store, graph, "FanIn")); - expect([...got.keys()].sort()).toEqual(["x", "y"]); - expect(got.get("x")?.map((e) => e.signal)).toEqual(["src-a", "src-b"]); - expect(got.get("y")?.map((e) => e.signal)).toEqual(["src-a"]); - }); -}); - -describe("readCellUpdates — multi-cell pipeline integration", () => { - it("propagates per-URI state through a 3-stage pipeline and reaches an empty-diff equilibrium", async () => { - // Pipeline: sources → Extractor (extracted) → Summarizer (summarized) → MetaExtractor (meta) - const graph = new DataflowGraph([ - { id: "Extractor", inputs: ["sources"], outputs: ["extracted"] }, - { id: "Summarizer", inputs: ["extracted"], outputs: ["summarized"] }, - { id: "MetaExtractor", inputs: ["summarized"], outputs: ["meta"] }, - ]); - const store = new InMemoryUpdatesStore(); - await store.setUpdates([ - { signal: "sources", uri: "doc-a", stamp: 1 }, - { signal: "sources", uri: "doc-b", stamp: 2 }, - ]); - - // Sweep helper: each cell handles every yielded input update (advances - // its per-cell watermark) and announces the same stamp on its output - // signal so the next cell sees it. - async function sweepOnce(): Promise> { - const processed: Record = {}; - for (const cellId of ["Extractor", "Summarizer", "MetaExtractor"]) { - const list: string[] = []; - const output = graph.getCellOutputs(cellId)[0] as string; - for await (const entry of readCellUpdates(store, graph, cellId)) { - list.push(entry.uri); - await store.handleUpdate({ - signal: entry.signal, - uri: entry.uri, - cell: cellId, - stamp: entry.stamp, - }); - await store.setUpdate({ signal: output, uri: entry.uri, stamp: entry.stamp }); - } - processed[cellId] = list; - } - return processed; - } - - const first = await sweepOnce(); - expect(first.Extractor).toEqual(["doc-a", "doc-b"]); - expect(first.Summarizer).toEqual(["doc-a", "doc-b"]); - expect(first.MetaExtractor).toEqual(["doc-a", "doc-b"]); - - const second = await sweepOnce(); - expect(second).toEqual({ Extractor: [], Summarizer: [], MetaExtractor: [] }); - - // Restamp a single source file. The cascade re-fires for that URI only. - await store.setUpdate({ signal: "sources", uri: "doc-a", stamp: 9 }); - const third = await sweepOnce(); - expect(third).toEqual({ - Extractor: ["doc-a"], - Summarizer: ["doc-a"], - MetaExtractor: ["doc-a"], - }); - - const fourth = await sweepOnce(); - expect(fourth).toEqual({ Extractor: [], Summarizer: [], MetaExtractor: [] }); - }); -}); diff --git a/packages/shared-dataflow/tests/updates-manager.test.ts b/packages/shared-dataflow/tests/updates-manager.test.ts deleted file mode 100644 index f95502b..0000000 --- a/packages/shared-dataflow/tests/updates-manager.test.ts +++ /dev/null @@ -1,703 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DataflowGraph } from "../src/dataflow-graph.js"; -import { InMemoryTransactionStore } from "../src/in-memory-transaction-store.js"; -import { UpdatesManager } from "../src/updates-manager.js"; - -describe("UpdatesManager — tracer", () => { - it("invokes a cell's handler with updateId=0 and the new transactionId, recording the tx on success", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - const seen: Array<{ updateId: number; transactionId: number }> = []; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async (params) => { - seen.push(params); - return true; - }, - }, - }); - - await manager.exec({ signals: ["s"] }); - - expect(seen).toEqual([{ updateId: 0, transactionId: 1 }]); - expect(await store.getCellTransaction("A")).toBe(1); - }); -}); - -describe("UpdatesManager — topological execution", () => { - it("runs A→B→C in topological order when seeds reach A", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const order: string[] = []; - - const make = (id: string) => async () => { - order.push(id); - return true; - }; - const manager = new UpdatesManager({ - graph, - store, - handlers: { A: make("A"), B: make("B"), C: make("C") }, - }); - - await manager.exec({ signals: ["s"] }); - - expect(order).toEqual(["A", "B", "C"]); - expect(await store.getCellTransaction("A")).toBe(1); - expect(await store.getCellTransaction("B")).toBe(1); - expect(await store.getCellTransaction("C")).toBe(1); - }); - - it("catches handler exceptions, treats them as false, and forwards to onError", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const errors: Array<{ cellId: string; error: unknown }> = []; - let bRan = false; - const boom = new Error("boom"); - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - throw boom; - }, - B: async () => { - bRan = true; - return true; - }, - }, - onError: (cellId, error) => errors.push({ cellId, error }), - }); - - await expect(manager.exec({ signals: ["s"] })).resolves.toBeUndefined(); // does not propagate - expect(await store.getCellTransaction("A")).toBe(0); // not recorded - expect(errors).toEqual([{ cellId: "A", error: boom }]); // onError called once with the throw - expect(bRan).toBe(true); // downstream still runs - }); - - it("does not record tx for a cell whose handler returned false; downstream cells still run", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - let bRan = false; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => false, // fails - B: async () => { - bRan = true; - return true; - }, - }, - }); - - await manager.exec({ signals: ["s"] }); - - expect(await store.getCellTransaction("A")).toBe(0); // not recorded - expect(bRan).toBe(true); // downstream still runs - expect(await store.getCellTransaction("B")).toBe(1); // B succeeded - }); - - it("passes the prior successful tx as updateId on subsequent activations", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - const seenUpdateIds: number[] = []; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async ({ updateId }) => { - seenUpdateIds.push(updateId); - return true; - }, - }, - }); - - await manager.exec({ signals: ["s"] }); // tx=1 - await manager.exec({ signals: ["s"] }); // tx=2 - await manager.exec({ signals: ["s"] }); // tx=3 - - expect(seenUpdateIds).toEqual([0, 1, 2]); - expect(await store.getCellTransaction("A")).toBe(3); - }); - - it("invokes a cell exactly once even when multiple seeds reach it", async () => { - // A reads S1 and S2; if both are in seeds, A must still run only once. - const graph = new DataflowGraph([ - { id: "A", inputs: ["s1", "s2"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - let aCalls = 0; - let bCalls = 0; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - aCalls++; - return true; - }, - B: async () => { - bCalls++; - return true; - }, - }, - }); - - await manager.exec({ signals: ["s1", "s2"] }); - - expect(aCalls).toBe(1); - expect(bCalls).toBe(1); - }); - - it("rejects a second run() while one is already in flight", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - let release!: () => void; - const gate = new Promise((res) => { - release = res; - }); - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - await gate; // hold the activation open - return true; - }, - }, - }); - - const first = manager.exec({ signals: ["s"] }); - await expect(manager.exec({ signals: ["s"] })).rejects.toThrow( - /already in progress|already running|in flight/i, - ); - release(); - await first; // first run still completes cleanly - expect(await store.getCellTransaction("A")).toBe(1); - }); - - it("can be re-invoked after the previous activation has finished", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - let calls = 0; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - calls++; - return true; - }, - }, - }); - - await manager.exec({ signals: ["s"] }); - await manager.exec({ signals: ["s"] }); - expect(calls).toBe(2); - }); - - it("silently skips cells that have no registered handler; downstream cells still run", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, // no handler registered - { id: "C", inputs: ["y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const order: string[] = []; - - const make = (id: string) => async () => { - order.push(id); - return true; - }; - const manager = new UpdatesManager({ - graph, - store, - handlers: { A: make("A"), C: make("C") }, // B intentionally absent - }); - - await manager.exec({ signals: ["s"] }); - - expect(order).toEqual(["A", "C"]); - expect(await store.getCellTransaction("A")).toBe(1); - expect(await store.getCellTransaction("B")).toBe(0); // never recorded - expect(await store.getCellTransaction("C")).toBe(1); - }); - - it("does nothing when called with no seeds and the graph has no probers", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: [] }, // no probers - ]); - const store = new InMemoryTransactionStore(); - let called = false; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - called = true; - return true; - }, - }, - }); - - await manager.exec(); - - expect(called).toBe(false); - expect(await store.getCellTransaction("A")).toBe(0); - // tx counter still advances on activation start - expect(await store.newTransactionId()).toBe(2); - }); - - it("runs all probers (cells with no inputs) and their cascade when called with no seeds", async () => { - const graph = new DataflowGraph([ - { id: "P1", inputs: [], outputs: ["x"] }, // prober - { id: "P2", inputs: [], outputs: ["y"] }, // prober - { id: "C", inputs: ["x", "y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const order: string[] = []; - - const make = (id: string) => async () => { - order.push(id); - return true; - }; - const manager = new UpdatesManager({ - graph, - store, - handlers: { P1: make("P1"), P2: make("P2"), C: make("C") }, - }); - - await manager.exec(); - - expect(new Set(order)).toEqual(new Set(["P1", "P2", "C"])); - // C must come after both probers - const idx = (id: string) => order.indexOf(id); - expect(idx("C")).toBeGreaterThan(idx("P1")); - expect(idx("C")).toBeGreaterThan(idx("P2")); - expect(await store.getCellTransaction("P1")).toBe(1); - expect(await store.getCellTransaction("P2")).toBe(1); - expect(await store.getCellTransaction("C")).toBe(1); - }); - - it("keeps updateId = last *successful* tx across a failed activation", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - const seenUpdateIds: number[] = []; - let attempt = 0; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async ({ updateId }) => { - seenUpdateIds.push(updateId); - attempt += 1; - // Succeed on attempt 1, fail on 2, succeed on 3. - return attempt !== 2; - }, - }, - }); - - await manager.exec({ signals: ["s"] }); // tx=1, success → record 1 - await manager.exec({ signals: ["s"] }); // tx=2, FAIL → record stays at 1 - await manager.exec({ signals: ["s"] }); // tx=3, success → record 3 - - expect(seenUpdateIds).toEqual([0, 1, 1]); // updateId stays at 1 after failed run - expect(await store.getCellTransaction("A")).toBe(3); - }); - - it("does not mistake Object.prototype methods for handlers when cell ids collide with them", async () => { - // Cell ids matching Object.prototype property names (`constructor`, - // `toString`, ...) must NOT pick up the prototype method as a handler. - // Without an own-property handler registered, the cell must be skipped. - const graph = new DataflowGraph([ - { id: "constructor", inputs: ["s"], outputs: ["x"] }, - { id: "toString", inputs: ["x"], outputs: ["y"] }, - { id: "hasOwnProperty", inputs: ["y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const errors: Array<{ cellId: string; error: unknown }> = []; - - const manager = new UpdatesManager({ - graph, - store, - handlers: {}, // no handlers at all — every cell must be silently skipped - onError: (cellId, error) => errors.push({ cellId, error }), - }); - - await manager.exec({ signals: ["s"] }); - - expect(await store.getCellTransaction("constructor")).toBe(0); - expect(await store.getCellTransaction("toString")).toBe(0); - expect(await store.getCellTransaction("hasOwnProperty")).toBe(0); - expect(errors).toEqual([]); - }); - - it("survives a throwing onError callback and continues running remaining cells", async () => { - // onError is documented as a passive notifier ("exception is otherwise - // swallowed") — a buggy logger that itself throws must not abort the run - // and strand mid-activation state. - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - let bRan = false; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - throw new Error("A failed"); - }, - B: async () => { - bRan = true; - return true; - }, - }, - onError: () => { - throw new Error("logger is broken"); - }, - }); - - await expect(manager.exec({ signals: ["s"] })).resolves.toBeUndefined(); - expect(bRan).toBe(true); - expect(await store.getCellTransaction("B")).toBe(1); - }); - - it("allocates one transactionId per activation; all cells share it", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const seenTx: Record = { A: [], B: [] }; - - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async ({ transactionId }) => { - seenTx.A?.push(transactionId); - return true; - }, - B: async ({ transactionId }) => { - seenTx.B?.push(transactionId); - return true; - }, - }, - }); - - await manager.exec({ signals: ["s"] }); - - expect(seenTx.A).toHaveLength(1); - expect(seenTx.B).toHaveLength(1); - expect(seenTx.A?.[0]).toBe(seenTx.B?.[0]); - }); -}); - -describe("UpdatesManager — run() as async generator", () => { - it("yields a begin event, one call event per executed cell in topological order, then an end event", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => true, - B: async () => true, - C: async () => true, - }, - }); - - const stages = []; - for await (const stage of manager.run({ signals: ["s"] })) { - stages.push(stage); - } - - // begin, A, B, C, end — all sharing the same transactionId - expect(stages[0]).toEqual({ type: "begin", transactionId: 1 }); - expect(stages.at(-1)).toEqual({ type: "end", transactionId: 1 }); - - const calls = stages.filter((s) => s.type === "call"); - expect(calls.map((c) => c.cellId)).toEqual(["A", "B", "C"]); - for (const call of calls) { - expect(call.transactionId).toBe(1); - expect(call.updateId).toBe(0); - expect(call.result).toBe(true); - } - }); - - it("reports `result: false` on a handler that returned false, and `result: false` on a handler that threw", async () => { - const graph = new DataflowGraph([ - { id: "Falsey", inputs: ["s"], outputs: [] }, - { id: "Thrower", inputs: ["s"], outputs: [] }, - { id: "Ok", inputs: ["s"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const manager = new UpdatesManager({ - graph, - store, - handlers: { - Falsey: async () => false, - Thrower: async () => { - throw new Error("nope"); - }, - Ok: async () => true, - }, - onError: () => { - /* swallow */ - }, - }); - - const byCell = new Map(); - for await (const stage of manager.run({ signals: ["s"] })) { - if (stage.type === "call") byCell.set(stage.cellId, stage.result); - } - - expect(byCell.get("Falsey")).toBe(false); - expect(byCell.get("Thrower")).toBe(false); - expect(byCell.get("Ok")).toBe(true); - }); - - it("the call event's `updateId` carries the cell's prior successful tx (and 0 on first activation)", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - const manager = new UpdatesManager({ - graph, - store, - handlers: { A: async () => true }, - }); - - const firstStages = []; - for await (const stage of manager.run({ signals: ["s"] })) firstStages.push(stage); - const firstCall = firstStages.find((s) => s.type === "call"); - expect(firstCall?.updateId).toBe(0); - - const secondStages = []; - for await (const stage of manager.run({ signals: ["s"] })) secondStages.push(stage); - const secondCall = secondStages.find((s) => s.type === "call"); - expect(secondCall?.updateId).toBe(1); // the prior successful tx - }); - - it("pauses between yields — the caller can drive the activation one stage at a time", async () => { - // Each cell's handler bumps a side-effect counter. We step the generator - // manually and verify the side effects happen exactly when we advance. - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - let aRan = 0; - let bRan = 0; - let cRan = 0; - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - aRan++; - return true; - }, - B: async () => { - bRan++; - return true; - }, - C: async () => { - cRan++; - return true; - }, - }, - }); - - const it = manager.run({ signals: ["s"] }); - - // First yield: begin. No handler has run yet. - let next = await it.next(); - expect(next.value).toEqual({ type: "begin", transactionId: 1 }); - expect(aRan).toBe(0); - expect(bRan).toBe(0); - expect(cRan).toBe(0); - - // Second yield: A's call event. A ran. B/C did NOT (we haven't advanced). - next = await it.next(); - expect(next.value).toMatchObject({ type: "call", cellId: "A", result: true }); - expect(aRan).toBe(1); - expect(bRan).toBe(0); - expect(cRan).toBe(0); - - // Third yield: B's call event. - next = await it.next(); - expect(next.value).toMatchObject({ type: "call", cellId: "B", result: true }); - expect(bRan).toBe(1); - expect(cRan).toBe(0); - - // Fourth yield: C's call event. - next = await it.next(); - expect(next.value).toMatchObject({ type: "call", cellId: "C", result: true }); - expect(cRan).toBe(1); - - // Fifth yield: end event. - next = await it.next(); - expect(next.value).toEqual({ type: "end", transactionId: 1 }); - - // Done. - next = await it.next(); - expect(next.done).toBe(true); - }); - - it("caller can stop iterating early via generator.return() — finally still releases the in-flight guard", async () => { - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - let bRan = false; - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => true, - B: async () => { - bRan = true; - return true; - }, - }, - }); - - const it = manager.run({ signals: ["s"] }); - await it.next(); // begin - await it.next(); // A's call - await it.return(undefined); // abandon iteration - - // B never ran — we closed the generator before it was reached. - expect(bRan).toBe(false); - // The in-flight guard is released — a fresh exec can start. - await expect(manager.exec({ signals: ["s"] })).resolves.toBeUndefined(); - expect(bRan).toBe(true); - }); - - it("can restart from explicit cell ids — runs those cells and their downstream cascade", async () => { - // Failed-cell restart scenario: A's handler returns false on the first - // activation; on a fresh activation seeded with `{ cells: ["A"] }`, A and - // its downstream B should run again. - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: [] }, - { id: "C", inputs: ["other"], outputs: [] }, // unrelated, must NOT run - ]); - const store = new InMemoryTransactionStore(); - let aAttempts = 0; - let bAttempts = 0; - let cAttempts = 0; - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - aAttempts++; - return aAttempts > 1; // first attempt fails - }, - B: async () => { - bAttempts++; - return true; - }, - C: async () => { - cAttempts++; - return true; - }, - }, - }); - - // First activation via signal seed. - const incompleteCells: string[] = []; - for await (const stage of manager.run({ signals: ["s"] })) { - if (stage.type === "call" && !stage.result) incompleteCells.push(stage.cellId); - } - expect(incompleteCells).toEqual(["A"]); - expect(aAttempts).toBe(1); - expect(bAttempts).toBe(1); // B still ran (downstream of failed A) - expect(await store.getCellTransaction("A")).toBe(0); // not recorded - expect(await store.getCellTransaction("B")).toBe(1); // B succeeded - - // Restart from the failed cells. A succeeds this time; B re-runs as - // downstream. C is untouched. - await manager.exec({ cells: incompleteCells }); - expect(aAttempts).toBe(2); - expect(bAttempts).toBe(2); - expect(cAttempts).toBe(0); - expect(await store.getCellTransaction("A")).toBe(2); - }); - - it("cell-seeded run includes the seeded cells in topological order with the rest", async () => { - // Seed cells from different layers. The result must respect graph order. - const graph = new DataflowGraph([ - { id: "A", inputs: ["s"], outputs: ["x"] }, - { id: "B", inputs: ["x"], outputs: ["y"] }, - { id: "C", inputs: ["y"], outputs: [] }, - ]); - const store = new InMemoryTransactionStore(); - const order: string[] = []; - const make = (id: string) => async () => { - order.push(id); - return true; - }; - const manager = new UpdatesManager({ - graph, - store, - handlers: { A: make("A"), B: make("B"), C: make("C") }, - }); - - // Seed with A and C — B is in between. Forward propagation from A's - // outputs pulls B in; topo sort then places A → B → C. - await manager.exec({ cells: ["A", "C"] }); - expect(order).toEqual(["A", "B", "C"]); - }); - - it("cell-seeded run silently drops unknown cell ids", async () => { - const graph = new DataflowGraph([{ id: "A", inputs: ["s"], outputs: [] }]); - const store = new InMemoryTransactionStore(); - let aRan = false; - const manager = new UpdatesManager({ - graph, - store, - handlers: { - A: async () => { - aRan = true; - return true; - }, - }, - }); - - await manager.exec({ cells: ["A", "NotARealCell"] }); - expect(aRan).toBe(true); - }); -}); diff --git a/packages/shared-dataflow/tests/updates-store-integration.test.ts b/packages/shared-dataflow/tests/updates-store-integration.test.ts deleted file mode 100644 index 533e8d7..0000000 --- a/packages/shared-dataflow/tests/updates-store-integration.test.ts +++ /dev/null @@ -1,462 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DataflowGraph } from "../src/dataflow-graph.js"; -import { InMemoryTransactionStore } from "../src/in-memory-transaction-store.js"; -import { InMemoryUpdatesStore } from "../src/in-memory-updates-store.js"; -import { readCellUpdates } from "../src/read-cell-updates.js"; -import type { CellHandler } from "../src/updates-manager.js"; -import { UpdatesManager } from "../src/updates-manager.js"; -import type { UpdatesStore } from "../src/updates-store.js"; - -// ----------------------------------------------------------------------------- -// Test domain stores — what handlers actually mutate. The UpdatesStore only -// carries pointers; the data lives here. - -interface FileRecord { - body: string; - updatedAt: number; -} -type FilesStore = Map; -type ContentStore = Map; -type ChunksStore = Map; -type EmbeddingsStore = Map; -type IndexStore = Map; - -// ----------------------------------------------------------------------------- -// Handler factories. Each consumes via `readCellUpdates` (per-cell handled -// watermark), records consumption with `handleUpdate(input)`, and announces -// production with `setUpdate(output)` carrying the observed upstream stamp. - -const graph = new DataflowGraph([ - { id: "FilesScanner", inputs: ["scan"], outputs: ["files"] }, - { id: "Extractor", inputs: ["files"], outputs: ["content"] }, - { id: "Chunker", inputs: ["content"], outputs: ["chunks"] }, - { id: "Embedder", inputs: ["chunks"], outputs: ["embeddings"] }, - { id: "Indexer", inputs: ["content", "chunks", "embeddings"], outputs: [] }, - { id: "ContentRemover", inputs: ["files:removed"], outputs: ["content:removed"] }, - { id: "ChunksRemover", inputs: ["content:removed"], outputs: ["chunks:removed"] }, - { - id: "EmbeddingsRemover", - inputs: ["chunks:removed"], - outputs: ["embeddings:removed"], - }, - { id: "IndexRemover", inputs: ["embeddings:removed"], outputs: [] }, -]); - -// Scanner — prober at the top. Its `scan` input never carries updates; it -// observes the files domain directly and stamps `files` with the activation tx. -function newFilesScanner(deps: { files: FilesStore; updatesStore: UpdatesStore }): CellHandler { - const lastSeen = new Map(); - return async ({ transactionId }) => { - for (const [uri, file] of deps.files) { - const seen = lastSeen.get(uri) ?? 0; - if (file.updatedAt > seen) { - await deps.updatesStore.setUpdate({ signal: "files", uri, stamp: transactionId }); - lastSeen.set(uri, file.updatedAt); - } - } - return true; - }; -} - -function newExtractor(deps: { - files: FilesStore; - content: ContentStore; - updatesStore: UpdatesStore; -}): CellHandler { - return async () => { - for await (const entry of readCellUpdates(deps.updatesStore, graph, "Extractor")) { - const file = deps.files.get(entry.uri); - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: entry.uri, - cell: "Extractor", - stamp: entry.stamp, - }); - if (file === undefined) continue; - deps.content.set(entry.uri, file.body.toUpperCase()); - await deps.updatesStore.setUpdate({ signal: "content", uri: entry.uri, stamp: entry.stamp }); - } - return true; - }; -} - -function newChunker(deps: { - content: ContentStore; - chunks: ChunksStore; - updatesStore: UpdatesStore; -}): CellHandler { - return async () => { - for await (const entry of readCellUpdates(deps.updatesStore, graph, "Chunker")) { - const fileUri = entry.uri; - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: fileUri, - cell: "Chunker", - stamp: entry.stamp, - }); - const content = deps.content.get(fileUri); - if (content === undefined) continue; - const half = Math.ceil(content.length / 2); - const parts: ReadonlyArray = [content.slice(0, half), content.slice(half)]; - let i = 0; - for (const part of parts) { - const chunkUri = `${fileUri}#${i}`; - deps.chunks.set(chunkUri, part); - await deps.updatesStore.setUpdate({ signal: "chunks", uri: chunkUri, stamp: entry.stamp }); - i++; - } - } - return true; - }; -} - -function newEmbedder(deps: { - chunks: ChunksStore; - embeddings: EmbeddingsStore; - updatesStore: UpdatesStore; -}): CellHandler { - return async () => { - for await (const entry of readCellUpdates(deps.updatesStore, graph, "Embedder")) { - const chunkUri = entry.uri; - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: chunkUri, - cell: "Embedder", - stamp: entry.stamp, - }); - const chunk = deps.chunks.get(chunkUri); - if (chunk === undefined) continue; - deps.embeddings.set(chunkUri, [chunk.length, chunk.charCodeAt(0) ?? 0]); - await deps.updatesStore.setUpdate({ - signal: "embeddings", - uri: chunkUri, - stamp: entry.stamp, - }); - } - return true; - }; -} - -// Sink cell — consumes three input signals, produces nothing. Tracks -// consumption purely via `handleUpdate` (the old "first output watermark" -// requirement is gone). Indexes whenever an embedding update flows. -function newIndexer(deps: { - chunks: ChunksStore; - embeddings: EmbeddingsStore; - index: IndexStore; - updatesStore: UpdatesStore; -}): CellHandler { - return async () => { - for await (const entry of readCellUpdates(deps.updatesStore, graph, "Indexer")) { - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: entry.uri, - cell: "Indexer", - stamp: entry.stamp, - }); - if (entry.signal !== "embeddings") continue; - const chunkUri = entry.uri; - const embedding = deps.embeddings.get(chunkUri); - const chunk = deps.chunks.get(chunkUri); - if (embedding === undefined || chunk === undefined) continue; - deps.index.set(chunkUri, { embedding, content: chunk }); - } - return true; - }; -} - -// Deletion cells — consume a `:removed` signal, mutate their domain, emit the -// next tombstone, and `removeUpdate` the upstream creation pair (which cascades -// away every cell's handled row for that URI). -function newContentRemover(deps: { - content: ContentStore; - updatesStore: UpdatesStore; -}): CellHandler { - return async () => { - const removed: string[] = []; - for await (const entry of readCellUpdates(deps.updatesStore, graph, "ContentRemover")) { - const uri = entry.uri; - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri, - cell: "ContentRemover", - stamp: entry.stamp, - }); - deps.content.delete(uri); - removed.push(uri); - await deps.updatesStore.setUpdate({ - signal: "content:removed", - uri, - stamp: entry.stamp, - }); - } - await deps.updatesStore.removeUpdates( - removed.flatMap((uri) => [ - { signal: "files", uri }, - { signal: "files:removed", uri }, - ]), - ); - return true; - }; -} - -function newChunksRemover(deps: { chunks: ChunksStore; updatesStore: UpdatesStore }): CellHandler { - return async () => { - const consumedFileUris: string[] = []; - for await (const entry of readCellUpdates(deps.updatesStore, graph, "ChunksRemover")) { - const fileUri = entry.uri; - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: fileUri, - cell: "ChunksRemover", - stamp: entry.stamp, - }); - consumedFileUris.push(fileUri); - const chunkPrefix = `${fileUri}#`; - const removedChunkUris = [...deps.chunks.keys()].filter((c) => c.startsWith(chunkPrefix)); - for (const chunkUri of removedChunkUris) { - deps.chunks.delete(chunkUri); - await deps.updatesStore.setUpdate({ - signal: "chunks:removed", - uri: chunkUri, - stamp: entry.stamp, - }); - } - } - await deps.updatesStore.removeUpdates( - consumedFileUris.flatMap((uri) => [ - { signal: "content", uri }, - { signal: "content:removed", uri }, - ]), - ); - return true; - }; -} - -function newEmbeddingsRemover(deps: { - embeddings: EmbeddingsStore; - updatesStore: UpdatesStore; -}): CellHandler { - return async () => { - const consumedChunkUris: string[] = []; - for await (const entry of readCellUpdates(deps.updatesStore, graph, "EmbeddingsRemover")) { - const chunkUri = entry.uri; - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: chunkUri, - cell: "EmbeddingsRemover", - stamp: entry.stamp, - }); - deps.embeddings.delete(chunkUri); - consumedChunkUris.push(chunkUri); - await deps.updatesStore.setUpdate({ - signal: "embeddings:removed", - uri: chunkUri, - stamp: entry.stamp, - }); - } - await deps.updatesStore.removeUpdates( - consumedChunkUris.flatMap((uri) => [ - { signal: "chunks", uri }, - { signal: "chunks:removed", uri }, - ]), - ); - return true; - }; -} - -function newIndexRemover(deps: { index: IndexStore; updatesStore: UpdatesStore }): CellHandler { - return async () => { - const consumedChunkUris: string[] = []; - for await (const entry of readCellUpdates(deps.updatesStore, graph, "IndexRemover")) { - const chunkUri = entry.uri; - await deps.updatesStore.handleUpdate({ - signal: entry.signal, - uri: chunkUri, - cell: "IndexRemover", - stamp: entry.stamp, - }); - deps.index.delete(chunkUri); - consumedChunkUris.push(chunkUri); - } - await deps.updatesStore.removeUpdates( - consumedChunkUris.flatMap((uri) => [ - { signal: "embeddings", uri }, - { signal: "embeddings:removed", uri }, - ]), - ); - return true; - }; -} - -// ----------------------------------------------------------------------------- - -function buildPipeline() { - const files: FilesStore = new Map(); - const content: ContentStore = new Map(); - const chunks: ChunksStore = new Map(); - const embeddings: EmbeddingsStore = new Map(); - const index: IndexStore = new Map(); - const updatesStore = new InMemoryUpdatesStore(); - const txStore = new InMemoryTransactionStore(); - - const handlers: Record = { - FilesScanner: newFilesScanner({ files, updatesStore }), - Extractor: newExtractor({ files, content, updatesStore }), - Chunker: newChunker({ content, chunks, updatesStore }), - Embedder: newEmbedder({ chunks, embeddings, updatesStore }), - Indexer: newIndexer({ chunks, embeddings, index, updatesStore }), - ContentRemover: newContentRemover({ content, updatesStore }), - ChunksRemover: newChunksRemover({ chunks, updatesStore }), - EmbeddingsRemover: newEmbeddingsRemover({ embeddings, updatesStore }), - IndexRemover: newIndexRemover({ index, updatesStore }), - }; - - const manager = new UpdatesManager({ graph, store: txStore, handlers }); - - return { files, content, chunks, embeddings, index, updatesStore, txStore, manager }; -} - -async function collect(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -// ----------------------------------------------------------------------------- - -describe("UpdatesStore integration — pipeline anchor test (per-cell handled)", () => { - it("scan-driven update cascade: scan → files → content → chunks → embeddings → index", async () => { - const p = buildPipeline(); - - p.files.set("f1", { body: "hello world", updatedAt: 1 }); - p.files.set("f2", { body: "another file", updatedAt: 1 }); - - await p.manager.exec({ signals: ["scan"] }); - - expect(p.content.get("f1")).toBe("HELLO WORLD"); - expect(p.content.get("f2")).toBe("ANOTHER FILE"); - expect(new Set(p.chunks.keys())).toEqual(new Set(["f1#0", "f1#1", "f2#0", "f2#1"])); - expect(p.embeddings.has("f1#0")).toBe(true); - expect(p.embeddings.has("f2#1")).toBe(true); - expect(new Set(p.index.keys())).toEqual(new Set(["f1#0", "f1#1", "f2#0", "f2#1"])); - - const filesRows = await collect(p.updatesStore.readEntries({ signal: "files", since: 0 })); - expect(filesRows.map((e) => e.uri).sort()).toEqual(["f1", "f2"]); - const contentRows = await collect(p.updatesStore.readEntries({ signal: "content", since: 0 })); - expect(contentRows.map((e) => e.uri).sort()).toEqual(["f1", "f2"]); - const chunksRows = await collect(p.updatesStore.readEntries({ signal: "chunks", since: 0 })); - expect(chunksRows.map((e) => e.uri).sort()).toEqual(["f1#0", "f1#1", "f2#0", "f2#1"]); - const embRows = await collect(p.updatesStore.readEntries({ signal: "embeddings", since: 0 })); - expect(embRows.map((e) => e.uri).sort()).toEqual(["f1#0", "f1#1", "f2#0", "f2#1"]); - - // The sink Indexer recorded consumption via handleUpdate (no output signal). - expect( - await collect(p.updatesStore.readUpdates({ signal: "embeddings", cell: "Indexer" })), - ).toEqual([]); - - for (const cellId of ["FilesScanner", "Extractor", "Chunker", "Embedder", "Indexer"]) { - expect(await p.txStore.getCellTransaction(cellId)).toBeGreaterThan(0); - } - }); - - it("re-indexing: changing a file's content + timestamp re-runs the full cascade", async () => { - const p = buildPipeline(); - - p.files.set("f1", { body: "hello world", updatedAt: 1 }); - await p.manager.exec({ signals: ["scan"] }); - expect(p.content.get("f1")).toBe("HELLO WORLD"); - expect(p.chunks.get("f1#0")).toBe("HELLO "); - const txAfterFirstRun = await p.txStore.getCellTransaction("Indexer"); - expect(txAfterFirstRun).toBeGreaterThan(0); - - p.files.set("f1", { body: "good night", updatedAt: 2 }); - await p.manager.exec({ signals: ["scan"] }); - - expect(p.content.get("f1")).toBe("GOOD NIGHT"); - expect(p.chunks.get("f1#0")).toBe("GOOD "); - expect(p.chunks.get("f1#1")).toBe("NIGHT"); - expect(p.index.get("f1#0")?.content).toBe("GOOD "); - expect(p.index.get("f1#1")?.content).toBe("NIGHT"); - - for (const cellId of ["FilesScanner", "Extractor", "Chunker", "Embedder", "Indexer"]) { - expect(await p.txStore.getCellTransaction(cellId)).toBeGreaterThan(txAfterFirstRun); - } - }); - - it("deletion cascade removes domain rows and cleans up updates AND handled rows", async () => { - const p = buildPipeline(); - - p.files.set("f1", { body: "hello world", updatedAt: 1 }); - p.files.set("f2", { body: "another file", updatedAt: 1 }); - await p.manager.exec({ signals: ["scan"] }); - - p.files.delete("f1"); - const removalTx = await p.txStore.newTransactionId(); - await p.updatesStore.setUpdate({ signal: "files:removed", uri: "f1", stamp: removalTx }); - - await p.manager.exec({ signals: ["files:removed"] }); - - expect(p.content.has("f1")).toBe(false); - expect(p.chunks.has("f1#0")).toBe(false); - expect(p.embeddings.has("f1#1")).toBe(false); - expect(p.index.has("f1#0")).toBe(false); - - expect(p.content.get("f2")).toBe("ANOTHER FILE"); - expect(p.index.has("f2#0")).toBe(true); - - // No f1-related rows in EITHER relation of the snapshot. - const snap = p.updatesStore.snapshot(); - const f1Updates: string[] = []; - for (const [signal, rows] of Object.entries(snap.updates)) { - for (const uri of Object.keys(rows)) { - if (uri === "f1" || uri.startsWith("f1#")) f1Updates.push(`${signal}:${uri}`); - } - } - const f1Handled: string[] = []; - for (const [signal, cells] of Object.entries(snap.handled)) { - for (const [cell, rows] of Object.entries(cells)) { - for (const uri of Object.keys(rows)) { - if (uri === "f1" || uri.startsWith("f1#")) f1Handled.push(`${signal}/${cell}:${uri}`); - } - } - } - expect(f1Updates).toEqual([]); - expect(f1Handled).toEqual([]); - - // f2's creation rows survive. - expect(snap.updates.files?.f2).toBeDefined(); - expect(snap.updates.content?.f2).toBeDefined(); - expect(snap.updates.chunks?.["f2#0"]).toBeDefined(); - expect(snap.updates.embeddings?.["f2#1"]).toBeDefined(); - }); - - it("idempotent re-scan: with no file changes every handler reads nothing and mutates nothing", async () => { - const p = buildPipeline(); - - p.files.set("f1", { body: "hello world", updatedAt: 1 }); - await p.manager.exec({ signals: ["scan"] }); - - const contentBefore = new Map(p.content); - const chunksBefore = new Map(p.chunks); - const embeddingsBefore = new Map(p.embeddings); - const indexBefore = new Map(p.index); - const updatesBefore = p.updatesStore.snapshot(); - - await p.manager.exec({ signals: ["scan"] }); - - expect(p.content).toEqual(contentBefore); - expect(p.chunks).toEqual(chunksBefore); - expect(p.embeddings).toEqual(embeddingsBefore); - expect(p.index).toEqual(indexBefore); - expect(p.updatesStore.snapshot()).toEqual(updatesBefore); - - // Every cell is caught up: readUpdates yields nothing. - expect( - await collect(p.updatesStore.readUpdates({ signal: "files", cell: "Extractor" })), - ).toEqual([]); - expect( - await collect(p.updatesStore.readUpdates({ signal: "embeddings", cell: "Indexer" })), - ).toEqual([]); - }); -}); diff --git a/packages/shared-dataflow/tsconfig.json b/packages/shared-dataflow/tsconfig.json deleted file mode 100644 index 80fa227..0000000 --- a/packages/shared-dataflow/tsconfig.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "compilerOptions": { - "target": "ES2022", - "module": "Preserve", - "moduleResolution": "Bundler", - "lib": ["ESNext"], - "strict": true, - "skipLibCheck": true, - "verbatimModuleSyntax": true, - "resolveJsonModule": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "noUnusedLocals": true, - "noUnusedParameters": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedIndexedAccess": true, - "resolvePackageJsonExports": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "noEmit": true - }, - "include": ["./src", "./tests"], - "exclude": ["node_modules", "dist"] -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa3c095..fb34ca0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -124,21 +124,6 @@ importers: specifier: 'catalog:' version: 4.4.3 - packages/shared-dataflow: - devDependencies: - rimraf: - specifier: 'catalog:' - version: 6.1.3 - tsdown: - specifier: 'catalog:' - version: 0.21.10(typescript@6.0.3) - typescript: - specifier: 'catalog:' - version: 6.0.3 - vitest: - specifier: 'catalog:' - version: 4.1.6(@types/node@25.8.0)(vite@7.3.2(@types/node@25.8.0)(jiti@2.6.1)(tsx@4.21.0)) - packages/shared-generators: devDependencies: '@types/node': From 233a1515e8bff8787883868c1f4801892dfea484 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Mon, 3 Aug 2026 00:47:54 +0200 Subject: [PATCH 2/2] fix(shared-logger): self-typed process.env access (source-consumed by host-neutral pkgs; browser-safe) Guards process for consumers whose tsconfig lacks node types and for the browser (process absent). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/shared-logger/src/logger.adapter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/shared-logger/src/logger.adapter.ts b/packages/shared-logger/src/logger.adapter.ts index 78d4b7b..915c104 100644 --- a/packages/shared-logger/src/logger.adapter.ts +++ b/packages/shared-logger/src/logger.adapter.ts @@ -76,7 +76,11 @@ export const [getLogger, setLogger, removeLogger] = newAdapter } }).process + ?.env; + const logLevel = (env?.LOG_LEVEL ?? "info") as LoggerLevel; logger.level = logLevel; logger.info(`[backbone] Starting up with log level: ${logLevel}`); return logger;