feat: add sitemap parity check workflow and script

.github/workflows/sitemap-parity.yml

@@ -0,0 +1,114 @@
+name: Sitemap Parity
+
+# Ensures every URL currently listed in the production sitemap is still
+# reachable on the deployment that will be produced by this branch. Runs on:
+#   - pull_request  -> waits for the Netlify deploy-preview, then checks
+#   - push to main  -> waits for the Netlify production deploy, then checks
+#
+# In both cases the job blocks until Netlify posts a successful commit status
+# for the exact commit under test (Netlify integrates via the GitHub Commit
+# Status API, not the Deployments API), then runs the check against the URL
+# from that status. If any prod URL is unreachable on the new deploy, the
+# check fails and blocks the merge.
+
+on:
+  pull_request:
+    branches: ['main']
+  push:
+    branches: ['main']
+
+permissions:
+  contents: read
+
+concurrency:
+  group: sitemap-parity-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # For PRs, check out the PR head commit so the script matches what
+          # Netlify actually deployed. For pushes, this is the pushed commit.
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Wait for Netlify deployment
+        id: wait
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          # Total wait budget in seconds (default 15 min) and poll interval.
+          WAIT_TIMEOUT: '900'
+          WAIT_INTERVAL: '15'
+        run: |
+          set -euo pipefail
+          echo "Waiting for a Netlify commit status on commit $SHA ..."
+
+          # Netlify reports build status via the GitHub Commit Status API
+          # (context typically `netlify/<site-name>/deploy-preview`) rather
+          # than the Deployments API, so we poll that endpoint.
+          deadline=$(( $(date +%s) + WAIT_TIMEOUT ))
+          preview_url=""
+
+          while [ "$(date +%s)" -lt "$deadline" ]; do
+            status_json=$(gh api "repos/${{ github.repository }}/commits/$SHA/status")
+
+            # Grab the first status whose context looks like a Netlify one.
+            # `map(select(...)) | .[0]` normalises the jq stream into a single
+            # (possibly null) object so downstream extraction is unambiguous.
+            netlify_state=$(printf '%s' "$status_json" | jq -r '
+              .statuses
+              | map(select(.context | ascii_downcase | startswith("netlify/")))
+              | .[0].state // ""')
+            netlify_target=$(printf '%s' "$status_json" | jq -r '
+              .statuses
+              | map(select(.context | ascii_downcase | startswith("netlify/")))
+              | .[0].target_url // ""')
+
+            if [ "$netlify_state" = "success" ]; then
+              if [ -z "$netlify_target" ] || [ "$netlify_target" = "null" ]; then
+                echo "::error::Netlify status is success but has no target_url."
+                exit 1
+              fi
+              preview_url="$netlify_target"
+              echo "Found successful Netlify deployment -> $preview_url"
+              break
+            elif [ "$netlify_state" = "failure" ] || [ "$netlify_state" = "error" ]; then
+              echo "::error::Netlify build reported state=$netlify_state for commit $SHA."
+              exit 1
+            elif [ -n "$netlify_state" ]; then
+              echo "Netlify status is currently: $netlify_state. Waiting ${WAIT_INTERVAL}s ..."
+            else
+              echo "No Netlify status posted yet. Waiting ${WAIT_INTERVAL}s ..."
+            fi
+
+            sleep "$WAIT_INTERVAL"
+          done
+
+          if [ -z "$preview_url" ]; then
+            echo "::error::Timed out after ${WAIT_TIMEOUT}s waiting for a Netlify deployment on $SHA."
+            exit 1
+          fi
+
+          echo "preview_url=$preview_url" >> "$GITHUB_OUTPUT"
+
+      - name: Check sitemap parity
+        env:
+          PREVIEW_URL: ${{ steps.wait.outputs.preview_url }}
+        run: |
+          echo "Preview URL: $PREVIEW_URL"
+          node scripts/check-sitemap-parity.mjs \
+            --prod-base=https://open-elements.com \
+            --preview-base="$PREVIEW_URL" \
+            --concurrency=4 \
+            --retries=4

scripts/check-sitemap-parity.mjs

@@ -0,0 +1,264 @@
+#!/usr/bin/env node
+/**
+ * Verify that every URL currently listed in the production sitemap is still
+ * reachable on a preview deployment (typically a Netlify PR preview).
+ *
+ * How it works:
+ *   1. Downloads `${prodBase}/en/sitemap.xml` and `${prodBase}/de/sitemap.xml`
+ *      and extracts every `<loc>` URL.
+ *   2. For each URL, keeps only the path (+ query, if any) and requests
+ *      `${previewBase}${path}` with redirect following. A final 2xx response
+ *      is a pass; anything else (3xx-terminal, 4xx, 5xx, network error) is a
+ *      fail.
+ *   3. Also downloads the preview's own `/en/sitemap.xml` and `/de/sitemap.xml`
+ *      and reports which production paths are missing from them (informational
+ *      — a path can be missing from the preview sitemap and still reachable,
+ *      which the reachability check catches separately).
+ *
+ * Usage:
+ *   node scripts/check-sitemap-parity.mjs \
+ *     --preview-base=https://deploy-preview-42--my-site.netlify.app \
+ *     [--prod-base=https://open-elements.com] \
+ *     [--concurrency=8] \
+ *     [--timeout=15000]
+ *
+ * Exits 0 if every prod URL is reachable on the preview, 1 otherwise.
+ */
+
+const DEFAULT_PROD_BASE = 'https://open-elements.com';
+const LOCALES = ['en', 'de'];
+const USER_AGENT =
+  'Mozilla/5.0 (compatible; open-elements-sitemap-parity/1.0; +https://github.com/open-elements/open-elements-website)';
+const RETRY_STATUSES = new Set([403, 408, 425, 429, 500, 502, 503, 504]);
+
+function parseArgs(argv) {
+  const out = {
+    prodBase: DEFAULT_PROD_BASE,
+    previewBase: null,
+    concurrency: 4,
+    timeoutMs: 15000,
+    retries: 3,
+  };
+  for (const arg of argv.slice(2)) {
+    if (arg.startsWith('--prod-base=')) {
+      out.prodBase = arg.slice('--prod-base='.length);
+    } else if (arg.startsWith('--preview-base=')) {
+      out.previewBase = arg.slice('--preview-base='.length);
+    } else if (arg.startsWith('--concurrency=')) {
+      out.concurrency = Number(arg.slice('--concurrency='.length));
+    } else if (arg.startsWith('--timeout=')) {
+      out.timeoutMs = Number(arg.slice('--timeout='.length));
+    } else if (arg.startsWith('--retries=')) {
+      out.retries = Number(arg.slice('--retries='.length));
+    } else if (arg === '--help' || arg === '-h') {
+      out.help = true;
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  return out;
+}
+
+function stripTrailingSlash(url) {
+  return url.replace(/\/+$/, '');
+}
+
+async function fetchText(url, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      signal: controller.signal,
+      headers: { 'user-agent': USER_AGENT },
+    });
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status} ${res.statusText} for ${url}`);
+    }
+    return await res.text();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function extractLocs(xml) {
+  const locs = [];
+  const re = /<loc>([^<]+)<\/loc>/g;
+  let m;
+  while ((m = re.exec(xml)) !== null) {
+    locs.push(m[1].trim());
+  }
+  return locs;
+}
+
+function urlToPath(rawUrl) {
+  try {
+    const u = new URL(rawUrl);
+    return `${u.pathname}${u.search}`;
+  } catch {
+    return null;
+  }
+}
+
+async function loadSitemapPaths(base, timeoutMs) {
+  const paths = new Set();
+  for (const locale of LOCALES) {
+    const sitemapUrl = `${base}/${locale}/sitemap.xml`;
+    const xml = await fetchText(sitemapUrl, timeoutMs);
+    for (const loc of extractLocs(xml)) {
+      const p = urlToPath(loc);
+      if (p) paths.add(p);
+    }
+  }
+  return paths;
+}
+
+async function checkReachable(previewBase, pathAndQuery, timeoutMs, retries) {
+  const target = `${previewBase}${pathAndQuery}`;
+  let lastStatus = 0;
+  let lastError;
+
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    if (attempt > 0) {
+      const backoffMs = 500 * 2 ** (attempt - 1);
+      await new Promise(resolve => setTimeout(resolve, backoffMs));
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const res = await fetch(target, {
+        redirect: 'follow',
+        signal: controller.signal,
+        headers: { 'user-agent': USER_AGENT },
+      });
+      if (res.ok) {
+        return {
+          ok: true,
+          status: res.status,
+          finalUrl: res.url,
+          attempts: attempt + 1,
+        };
+      }
+      lastStatus = res.status;
+      if (!RETRY_STATUSES.has(res.status)) {
+        return { ok: false, status: res.status, attempts: attempt + 1 };
+      }
+    } catch (err) {
+      lastError = err.message ?? String(err);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  return {
+    ok: false,
+    status: lastStatus,
+    error: lastError,
+    attempts: retries + 1,
+  };
+}
+
+async function runWithConcurrency(items, concurrency, worker) {
+  const results = new Array(items.length);
+  let cursor = 0;
+  const runners = Array.from(
+    { length: Math.min(concurrency, items.length) },
+    async () => {
+      while (true) {
+        const i = cursor++;
+        if (i >= items.length) return;
+        results[i] = await worker(items[i], i);
+      }
+    },
+  );
+  await Promise.all(runners);
+  return results;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  if (args.help || !args.previewBase) {
+    console.error(
+      'Usage: node scripts/check-sitemap-parity.mjs --preview-base=<url> [--prod-base=<url>] [--concurrency=N] [--timeout=ms]',
+    );
+    process.exit(args.help ? 0 : 2);
+  }
+
+  const prodBase = stripTrailingSlash(args.prodBase);
+  const previewBase = stripTrailingSlash(args.previewBase);
+
+  console.log(`Prod base:    ${prodBase}`);
+  console.log(`Preview base: ${previewBase}`);
+  console.log(`Concurrency:  ${args.concurrency}`);
+  console.log('');
+
+  console.log('Downloading production sitemaps...');
+  const prodPaths = await loadSitemapPaths(prodBase, args.timeoutMs);
+  console.log(`  ${prodPaths.size} unique paths in production sitemap.`);
+
+  console.log('Downloading preview sitemaps (informational)...');
+  let previewPaths = new Set();
+  try {
+    previewPaths = await loadSitemapPaths(previewBase, args.timeoutMs);
+    console.log(`  ${previewPaths.size} unique paths in preview sitemap.`);
+  } catch (err) {
+    console.warn(
+      `  WARN: failed to fetch preview sitemap (${err.message ?? err}). ` +
+        'Continuing with reachability check only.',
+    );
+  }
+
+  const missingFromPreviewSitemap = [...prodPaths].filter(
+    p => previewPaths.size > 0 && !previewPaths.has(p),
+  );
+  if (missingFromPreviewSitemap.length > 0) {
+    console.log('');
+    console.log(
+      `Paths in prod sitemap but NOT in preview sitemap (${missingFromPreviewSitemap.length}):`,
+    );
+    for (const p of missingFromPreviewSitemap) console.log(`  - ${p}`);
+  }
+
+  console.log('');
+  console.log(`Checking reachability of ${prodPaths.size} paths on preview...`);
+
+  const pathList = [...prodPaths].sort();
+  const results = await runWithConcurrency(
+    pathList,
+    args.concurrency,
+    async pathAndQuery => {
+      const r = await checkReachable(
+        previewBase,
+        pathAndQuery,
+        args.timeoutMs,
+        args.retries,
+      );
+      return { pathAndQuery, ...r };
+    },
+  );
+
+  const failures = results.filter(r => !r.ok);
+  const okCount = results.length - failures.length;
+
+  console.log('');
+  console.log(`Reachable: ${okCount} / ${results.length}`);
+
+  if (failures.length > 0) {
+    console.log('');
+    console.log(`Broken paths (${failures.length}):`);
+    for (const f of failures) {
+      const detail =
+        f.error != null ? `error: ${f.error}` : `status: ${f.status}`;
+      console.log(`  ✗ ${f.pathAndQuery}  (${detail})`);
+    }
+    process.exit(1);
+  }
+
+  console.log('');
+  console.log('All production URLs are reachable on the preview. ✓');
+}
+
+main().catch(err => {
+  console.error(err.stack ?? err.message ?? err);
+  process.exit(1);
+});