From bd929a01602adafd099312443507bac9a408ce83 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 18 Jul 2026 16:13:41 -0500 Subject: [PATCH 1/2] feat: add route parity CI guard and MCP smoke probe OpenAPI-based route discovery fixes the parity test for included routers; compose smoke now validates MCP tools/list; nightly live Playwright workflow and parity doc sync complete the hardening pass. --- .github/workflows/nightly-e2e.yml | 71 +++++++++++++++++++++++++ apps/web/e2e/live-smoke.spec.ts | 20 +++++++ apps/web/package.json | 3 +- apps/web/playwright.live.config.ts | 18 +++++++ docs/feature-parity.md | 6 +++ docs/parity-audit.md | 8 +-- scripts/smoke.sh | 5 ++ tests/test_api_route_parity.py | 85 ++++++++++++++++++++++++++++++ 8 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/nightly-e2e.yml create mode 100644 apps/web/e2e/live-smoke.spec.ts create mode 100644 apps/web/playwright.live.config.ts create mode 100644 tests/test_api_route_parity.py diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml new file mode 100644 index 0000000..58f061b --- /dev/null +++ b/.github/workflows/nightly-e2e.yml @@ -0,0 +1,71 @@ +name: Nightly live-stack e2e + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +jobs: + live-e2e: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Start compose stack + env: + SYNTHORA_AUTH_MODE: none + SYNTHORA_SECRET_KEY: ci-nightly-e2e-secret-key-not-prod-32 + SYNTHORA_CHECKPOINT_BACKEND: postgres + OPENAI_API_KEY: "" + OLLAMA_BASE_URL: "" + SYNTHORA_EMBEDDINGS: hash + run: | + docker compose up -d --build + for i in $(seq 1 90); do + if curl -fsS "http://localhost:8000/health" >/dev/null 2>&1; then + break + fi + sleep 2 + done + curl -fsS "http://localhost:8000/health" + for i in $(seq 1 60); do + if curl -fsS "http://localhost:3000/" >/dev/null 2>&1; then + break + fi + sleep 2 + done + curl -fsS "http://localhost:3000/" >/dev/null + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: apps/web/package-lock.json + + - name: Install web deps + working-directory: apps/web + run: npm ci + + - name: Install Playwright Chromium + working-directory: apps/web + run: npx playwright install --with-deps chromium + + - name: Live-stack Playwright smoke + working-directory: apps/web + env: + PLAYWRIGHT_BASE_URL: http://localhost:3000 + CI: true + run: npx playwright test --config playwright.live.config.ts + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-live-report + path: apps/web/playwright-report/ + retention-days: 7 + + - name: Tear down compose + if: always() + run: docker compose down diff --git a/apps/web/e2e/live-smoke.spec.ts b/apps/web/e2e/live-smoke.spec.ts new file mode 100644 index 0000000..0855c14 --- /dev/null +++ b/apps/web/e2e/live-smoke.spec.ts @@ -0,0 +1,20 @@ +import { expect, test } from "@playwright/test"; + +/** + * Live-stack smoke: hits real API via compose web (no fetch/WebSocket mocks). + * Run after `docker compose up` with SYNTHORA_AUTH_MODE=none. + */ + +test("home loads pipelines from live API", async ({ page }) => { + await page.goto("/"); + await expect( + page.getByRole("heading", { name: /ask a research question/i }), + ).toBeVisible({ timeout: 30_000 }); + await expect(page.getByRole("radiogroup", { name: "pipeline" })).toBeVisible({ + timeout: 15_000, + }); + await expect(page.getByText("Fast research")).toBeVisible(); + await expect( + page.getByRole("button", { name: /start research/i }), + ).toBeVisible(); +}); diff --git a/apps/web/package.json b/apps/web/package.json index 6cf1af7..23b0ff1 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,8 @@ "build": "tsc -b && vite build", "preview": "vite preview", "test": "vitest run", - "test:e2e": "playwright test" + "test:e2e": "playwright test", + "test:e2e:live": "playwright test --config playwright.live.config.ts" }, "dependencies": { "react": "^18.3.1", diff --git a/apps/web/playwright.live.config.ts b/apps/web/playwright.live.config.ts new file mode 100644 index 0000000..e7f70f3 --- /dev/null +++ b/apps/web/playwright.live.config.ts @@ -0,0 +1,18 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** Live-stack e2e: requires compose (or dev) API + web already running. */ +export default defineConfig({ + testDir: "./e2e", + testMatch: "live-smoke.spec.ts", + timeout: 90_000, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + reporter: process.env.CI ? [["github"], ["html", { open: "never" }]] : "list", + use: { + baseURL: process.env.PLAYWRIGHT_BASE_URL || "http://127.0.0.1:3000", + trace: "retain-on-failure", + screenshot: "only-on-failure", + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], +}); diff --git a/docs/feature-parity.md b/docs/feature-parity.md index 914cc8d..b705d9b 100644 --- a/docs/feature-parity.md +++ b/docs/feature-parity.md @@ -145,6 +145,12 @@ overlay wires ``OLLAMA_BASE_URL`` for the ollama profile; async ``iter_run_events`` handles ``ConnectionClosed`` like sync; README/smoke comments updated. +Closed on ``feat/route-parity-guard`` (PR #15): CI regression test +(``tests/test_api_route_parity.py``) discovers routes via OpenAPI and asserts +every ``/api/v1`` REST path appears in sync SDK, async SDK, and web ``api.ts``; +compose smoke probes MCP REST ``tools/list``; nightly live-stack Playwright +workflow added. + No known functional gaps remain beyond explicit non-goals below. Chat remains session-scoped ``fast_research`` with prior-report memory — diff --git a/docs/parity-audit.md b/docs/parity-audit.md index b51a8c3..c980ec1 100644 --- a/docs/parity-audit.md +++ b/docs/parity-audit.md @@ -1,7 +1,7 @@ # Synthora parity audit checklist Living checklist against Open Deep Research, STORM/Co-STORM, and Local Deep -Research. Last verified on `main` after PR #13 (`8e13595`) and ollama-profile overlay. +Research. Last verified on `main` after PR #15 (`feat/route-parity-guard`). ## Open Deep Research @@ -61,6 +61,7 @@ Research. Last verified on `main` after PR #13 (`8e13595`) and ollama-profile ov | RAG collection engine scoped to run workspace (contextvar) | done | | WebSocket auth: token required in session mode, foreign runs rejected | done | | MCP outbound URL SSRF guard (`SYNTHORA_MCP_ALLOWLIST`) | done | +| MCP inbound DNS rebinding protection (optional) | done | | Boot refusal on insecure secret key in session mode | done | | Durable Postgres checkpointer (compose default `postgres`) | done | @@ -70,8 +71,9 @@ Research. Last verified on `main` after PR #13 (`8e13595`) and ollama-profile ov |---|---| | Compose default embeddings (`SYNTHORA_EMBEDDINGS=hash`) without Ollama profile | done (PR #13) | | Ollama profile overlay (`docker-compose.ollama.yml` sets service URL) | done (PR #14) | -| Live compose smoke (`scripts/smoke.sh`: research, export, upload) | done | -| Playwright UI e2e | API-mocked (CI speed); not a live-stack browser gate | +| Live compose smoke (`scripts/smoke.sh`: research, export, upload, MCP list) | done | +| Route parity CI guard (`tests/test_api_route_parity.py`) | done (PR #15) | +| Playwright UI e2e | API-mocked (CI speed); live-stack browser gate (nightly) | ## Residual gaps (deliberate, not silent) diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 63239c7..b8b02eb 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -131,4 +131,9 @@ fi UPLOAD=$(cat /tmp/synthora-upload.json) echo "$UPLOAD" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert d.get("id"), d; print("upload ok:", d["id"])' +echo "==> MCP REST tools/list" +curl -fsS -X POST "http://localhost:${SYNTHORA_API_PORT:-8000}/api/v1/mcp/tools/list" \ + -H 'Content-Type: application/json' -d '{}' \ + | python3 -c "import json,sys; names={t['name'] for t in json.load(sys.stdin)['tools']}; expected={'start_research','get_run_status','get_report','search_documents'}; assert names==expected, names" + echo "smoke test passed" diff --git a/tests/test_api_route_parity.py b/tests/test_api_route_parity.py new file mode 100644 index 0000000..4a950a7 --- /dev/null +++ b/tests/test_api_route_parity.py @@ -0,0 +1,85 @@ +"""CI guard: REST /api/v1 routes must appear in SDK + web client sources.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import synthora.api.main as api_main + +ROOT = Path(__file__).resolve().parents[1] +SDK_SYNC = ROOT / "packages/sdk/src/synthora/sdk/client.py" +SDK_ASYNC = ROOT / "packages/sdk/src/synthora/sdk/async_client.py" +WEB_API = ROOT / "apps/web/src/api.ts" + + +def _api_v1_paths() -> set[str]: + """REST paths from OpenAPI (includes routes mounted via include_router).""" + paths = { + path + for path in api_main.app.openapi()["paths"] + if path.startswith("/api/v1") + } + return paths + + +def _normalize_for_source_match(path: str) -> str: + """Turn ``/api/v1/research/{run_id}/report`` into a regex fragment.""" + escaped = re.escape(path) + escaped = re.sub(r"\\{[^}]+\\}", r"[^/\"'`]+", escaped) + return escaped + + +def _source_covers(path: str, source: str) -> bool: + pattern = _normalize_for_source_match(path) + return re.search(pattern, source) is not None + + +def test_api_routes_exist_in_fastapi(): + paths = _api_v1_paths() + assert len(paths) >= 30 + assert "/api/v1/research" in paths + assert "/api/v1/mcp/tools/list" in paths + + +def test_sync_sdk_covers_all_api_v1_routes(): + source = SDK_SYNC.read_text(encoding="utf-8") + missing = [p for p in sorted(_api_v1_paths()) if not _source_covers(p, source)] + assert missing == [], f"sync SDK missing routes: {missing}" + + +def test_async_sdk_covers_all_api_v1_routes(): + source = SDK_ASYNC.read_text(encoding="utf-8") + missing = [p for p in sorted(_api_v1_paths()) if not _source_covers(p, source)] + assert missing == [], f"async SDK missing routes: {missing}" + + +def test_web_api_ts_covers_all_api_v1_routes(): + source = WEB_API.read_text(encoding="utf-8") + missing = [p for p in sorted(_api_v1_paths()) if not _source_covers(p, source)] + assert missing == [], f"web api.ts missing routes: {missing}" + + +def test_sync_and_async_sdk_public_surface_parity(): + """Async client should expose the same REST helpers as sync (except lifecycle).""" + sync_src = SDK_SYNC.read_text(encoding="utf-8") + async_src = SDK_ASYNC.read_text(encoding="utf-8") + + sync_methods = set(re.findall(r"^\s+def (\w+)\(", sync_src, re.MULTILINE)) + async_methods = set(re.findall(r"^\s+async def (\w+)\(", async_src, re.MULTILINE)) + + non_rest_helpers = { + "__init__", + "_headers", + "close", + "aclose", + "__aenter__", + "__aexit__", + "events_ws_url", + "export_url", + } + sync_only = sync_methods - async_methods - non_rest_helpers + async_only = async_methods - sync_methods - non_rest_helpers + + assert sync_only == set(), f"sync-only methods: {sync_only}" + assert async_only == set(), f"async-only methods: {async_only}" From e368adf09b0a47cdcb14b21c36dfeffc8b899193 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 18 Jul 2026 16:15:41 -0500 Subject: [PATCH 2/2] fix: exclude live-smoke from mocked Playwright CI run live-smoke.spec.ts requires a running compose stack and belongs only in the nightly live config, not the default mocked e2e job. --- apps/web/playwright.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts index 36a39d5..ea0b137 100644 --- a/apps/web/playwright.config.ts +++ b/apps/web/playwright.config.ts @@ -2,6 +2,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./e2e", + testIgnore: "live-smoke.spec.ts", timeout: 60_000, fullyParallel: false, forbidOnly: !!process.env.CI,