From 0df2d915fa714058994ed80eb8901680632f96f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 20:56:03 +0000 Subject: [PATCH 01/12] fix(ci): rescue daily-regen from GitHub's silently dropped cron ticks GitHub's scheduler has been dropping daily-regen.yml's schedule events in multi-day streaks since early June (e.g. only 5 scheduled runs in the last 13 days instead of ~130), while less-frequent crons in the same repo (watchdog-stuck-jobs, bot-serving-check) kept firing with their usual 2-3 h delay. Every run that did start succeeded, so nothing alarmed and the oldest specs simply stopped being refreshed. Two defences: - daily-regen's cron moves off the top of the hour (:00 -> :17), the documented high-load window in which GitHub delays/drops schedule events. - watchdog-stuck-jobs gains a cron-liveness section: if no daily-regen run (any trigger) has started for >10 h outside the 17-21 UTC Berlin-evening quiet window, it re-dispatches the workflow with default inputs, capping future starvation at roughly half a day. Also documents watchdog-stuck-jobs.yml in docs/workflows/overview.md, which had been missing from the workflow table. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/daily-regen.yml | 19 +++++++++--- .github/workflows/watchdog-stuck-jobs.yml | 37 ++++++++++++++++++++++- CHANGELOG.md | 8 +++++ docs/workflows/overview.md | 3 +- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index 4c628a73af..3f8e219fa9 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -4,10 +4,18 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # Picks the N oldest specs (by most-recent implementation `updated` timestamp) # and re-dispatches `bulk-generate.yml` for each. Default N=1 per cron tick. # -# Schedule: hourly, skipping the 20:00–23:59 Berlin (CEST) evening window. -# Berlin CEST evening hours 20, 21, 22, 23 map to UTC 18, 19, 20, 21 — these -# are intentionally skipped so runs never start during the user's evening. -# All other UTC hours run, giving 20 ticks per day. +# Schedule: every 2 hours, skipping the 20:00–23:59 Berlin (CEST) evening +# window. Berlin CEST evening hours 20, 21, 22, 23 map to UTC 18, 19, 20, 21 — +# these are intentionally skipped so runs never start during the user's +# evening. That gives 10 ticks per day. +# +# The tick fires at minute 17 (not :00) on purpose: GitHub's scheduler is +# most overloaded at the top of the hour and delays/drops `schedule` events +# then (github.com docs on `on.schedule`). This repo has seen multi-day +# streaks of silently dropped ticks for exactly this workflow while +# less-frequent crons kept firing. As a second line of defence, +# watchdog-stuck-jobs.yml re-dispatches this workflow when no run has +# started for >10 h outside the quiet window. # # bulk-generate is serialised via its own concurrency group. Default model is # Sonnet for higher implementation quality; can be overridden per manual run. @@ -15,10 +23,11 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # Triggers: # - schedule: 10× daily (UTC, every 2 hours, skipping the 18–21 UTC window) → Sonnet # - workflow_dispatch: manual, with inputs for spec id, model, count, dry-run +# (also used by the watchdog's cron-liveness rescue, with all defaults) on: schedule: - - cron: '0 0,2,4,6,8,10,12,14,16,22 * * *' + - cron: '17 0,2,4,6,8,10,12,14,16,22 * * *' workflow_dispatch: inputs: specification_id: diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 7d4b622b1a..5f3d12d6a8 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -1,7 +1,7 @@ name: "Watchdog: Stuck Jobs" run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manual)' || '(cron)' }}" -# Periodic safety net for the impl pipeline. Catches three failure modes +# Periodic safety net for the impl pipeline. Catches four failure modes # the regular workflows miss: # 1. PRs labeled `ai-review-failed` without `ai-review-rescued` # (impl-review-retry.yml normally handles these; watchdog covers @@ -10,6 +10,8 @@ run-name: "Watchdog: scan ${{ github.event_name == 'workflow_dispatch' && '(manu # (review handed off to repair, repair crashed, nothing else fired). # 3. spec-ready issues with `generate:` or `impl::failed` and # no open PR for that (spec, lib) pair after the staleness window. +# 4. daily-regen's cron silently starved by GitHub's scheduler (no run +# for >10 h outside the quiet window) — re-dispatched manually. # # Retries are bounded per-cause via marker labels: # - `ai-review-rescued` (review failures) @@ -295,4 +297,37 @@ jobs: done done < <(echo "$ISS_JSON" | jq -c '.[]') + ##### C) Cron-liveness: rescue a starved daily-regen schedule ###### + # GitHub silently drops `schedule` ticks for daily-regen (the + # repo's most frequent cron) during scheduler overload — observed + # as multi-day gaps while this watchdog and bot-serving-check kept + # firing. daily-regen ticks are ≤6 h apart (2 h cadence plus the + # 18–21 UTC quiet window), so a >10 h silence means the schedule + # is starved; re-dispatch it manually with default inputs. + # Hours 17–21 UTC are skipped so the rescue never launches a + # pipeline into the Berlin-evening quiet window the cron itself + # deliberately avoids (17 included because bulk-generate starts + # immediately after dispatch and would spill into it). + LIVENESS_HOURS=10 + HOUR=$(date -u +%-H) + if (( HOUR >= 17 && HOUR <= 21 )); then + echo "::notice::daily-regen liveness: inside/adjacent to quiet window (UTC hour $HOUR) — check skipped" + else + LAST=$(gh run list --workflow daily-regen.yml --limit 1 \ + --json startedAt --jq '.[0].startedAt // empty') + if [[ -z "$LAST" ]]; then + echo "::warning::daily-regen liveness: no runs found — skipping rescue" + else + last_sec=$(date -u -d "$LAST" +%s) + gap=$(( NOW - last_sec )) + gap_h=$(( gap / 3600 )) + if (( gap > LIVENESS_HOURS * 3600 )); then + echo "::warning::daily-regen last started ${gap_h}h ago (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" + dispatch "daily-regen: cron starved (${gap_h}h silent)" daily-regen.yml + else + echo "daily-regen liveness: last run started ${gap_h}h ago — healthy" + fi + fi + fi + echo "::notice::Watchdog scan complete" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3491012050..bffb406d46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,6 +129,14 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed +- **Scheduled spec regeneration no longer silently starves for days** — GitHub's scheduler + had been dropping `daily-regen.yml`'s cron ticks in multi-day streaks since early June + (last 13 days: only 5 scheduled runs instead of ~130) while less-frequent crons in the same + repo kept firing; every run that did start was green, so nothing alarmed. The cron now fires + at :17 instead of :00 (GitHub documents top-of-hour scheduler overload as a delay/drop + cause), and `watchdog-stuck-jobs.yml` gained a cron-liveness rescue that re-dispatches + daily-regen whenever no run has started for >10 h outside the Berlin-evening quiet window, + capping any future starvation at roughly half a day. - **Tag search uses the GIN index and stops treating `%`/`_` as wildcards** — `SpecRepository.search_by_tags` cast the JSONB `tags` column to text and ran LIKE, which the `ix_specs_tags` GIN index can never serve (sequential scan on every MCP tag search) and which diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index d42f3ba920..6f03003b52 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -167,7 +167,8 @@ Located in `.github/workflows/`: | `impl-repair.yml` | Fixes rejected implementations | | `impl-merge.yml` | Merges approved PRs | | `bulk-generate.yml` | Batch implementation generation | -| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations | +| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (fires at :17 past even UTC hours to dodge GitHub's top-of-hour scheduler overload) | +| `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews/repairs/merges/generations and rescues daily-regen when its cron is silently starved by GitHub (>10 h without a run) | | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges | From 38bac9977b2a476f670d56b445e64560c71c8414 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 20:56:38 +0000 Subject: [PATCH 02/12] docs(changelog): add PR ref #9649 to the daily-regen fix entry Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bffb406d46..1864780dfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,7 +136,7 @@ aggregate instead: an italic *Catalog* line at the end of the version section an at :17 instead of :00 (GitHub documents top-of-hour scheduler overload as a delay/drop cause), and `watchdog-stuck-jobs.yml` gained a cron-liveness rescue that re-dispatches daily-regen whenever no run has started for >10 h outside the Berlin-evening quiet window, - capping any future starvation at roughly half a day. + capping any future starvation at roughly half a day (#9649). - **Tag search uses the GIN index and stops treating `%`/`_` as wildcards** — `SpecRepository.search_by_tags` cast the JSONB `tags` column to text and ran LIKE, which the `ix_specs_tags` GIN index can never serve (sequential scan on every MCP tag search) and which From 3a0b1538b8e140c2a3251c56354f63f15c598e91 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 20:59:41 +0000 Subject: [PATCH 03/12] fix(watchdog): scope daily-regen liveness to main, use createdAt Review feedback: a manual daily-regen run on a non-default branch must not mask a starved main schedule, and startedAt is null while the newest run is still queued (which the previous query misread as 'no runs found'). Scope gh run list to --branch main and measure the gap from createdAt, which is always populated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/daily-regen.yml | 4 ++-- .github/workflows/watchdog-stuck-jobs.yml | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index 3f8e219fa9..b7c2c2d857 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -14,8 +14,8 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # then (github.com docs on `on.schedule`). This repo has seen multi-day # streaks of silently dropped ticks for exactly this workflow while # less-frequent crons kept firing. As a second line of defence, -# watchdog-stuck-jobs.yml re-dispatches this workflow when no run has -# started for >10 h outside the quiet window. +# watchdog-stuck-jobs.yml re-dispatches this workflow when main has seen +# no new run for >10 h outside the quiet window. # # bulk-generate is serialised via its own concurrency group. Default model is # Sonnet for higher implementation quality; can be overridden per manual run. diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 5f3d12d6a8..7fbc3ce947 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -313,19 +313,23 @@ jobs: if (( HOUR >= 17 && HOUR <= 21 )); then echo "::notice::daily-regen liveness: inside/adjacent to quiet window (UTC hour $HOUR) — check skipped" else - LAST=$(gh run list --workflow daily-regen.yml --limit 1 \ - --json startedAt --jq '.[0].startedAt // empty') + # --branch main: schedule runs (and rescue dispatches) live on the + # default branch; a manual run on a feature branch must not mask a + # starved main schedule. createdAt (not startedAt): always set, + # even while the newest run is still queued. + LAST=$(gh run list --workflow daily-regen.yml --branch main --limit 1 \ + --json createdAt --jq '.[0].createdAt // empty') if [[ -z "$LAST" ]]; then - echo "::warning::daily-regen liveness: no runs found — skipping rescue" + echo "::warning::daily-regen liveness: no runs found on main — skipping rescue" else last_sec=$(date -u -d "$LAST" +%s) gap=$(( NOW - last_sec )) gap_h=$(( gap / 3600 )) if (( gap > LIVENESS_HOURS * 3600 )); then - echo "::warning::daily-regen last started ${gap_h}h ago (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" + echo "::warning::daily-regen: no new run on main for ${gap_h}h (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" dispatch "daily-regen: cron starved (${gap_h}h silent)" daily-regen.yml else - echo "daily-regen liveness: last run started ${gap_h}h ago — healthy" + echo "daily-regen liveness: newest run on main created ${gap_h}h ago — healthy" fi fi fi From b019af01cb41f333d46b7b63813faae499f8f916 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 21:03:44 +0000 Subject: [PATCH 04/12] chore(watchdog): log liveness gap as hours+minutes Review feedback: integer-hour truncation could print '10h (> 10h)' when the real gap is 10h59m. Log h+mm so the message matches the threshold evaluation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/watchdog-stuck-jobs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index 7fbc3ce947..d8faae0d14 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -324,12 +324,12 @@ jobs: else last_sec=$(date -u -d "$LAST" +%s) gap=$(( NOW - last_sec )) - gap_h=$(( gap / 3600 )) + gap_hm=$(printf '%dh%02dm' $(( gap / 3600 )) $(( (gap % 3600) / 60 ))) if (( gap > LIVENESS_HOURS * 3600 )); then - echo "::warning::daily-regen: no new run on main for ${gap_h}h (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" - dispatch "daily-regen: cron starved (${gap_h}h silent)" daily-regen.yml + echo "::warning::daily-regen: no new run on main for ${gap_hm} (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" + dispatch "daily-regen: cron starved (${gap_hm} silent)" daily-regen.yml else - echo "daily-regen liveness: newest run on main created ${gap_h}h ago — healthy" + echo "daily-regen liveness: newest run on main created ${gap_hm} ago — healthy" fi fi fi From a383a8ce0d9e7057108ba49f0bbf177fbe794375 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 21:07:10 +0000 Subject: [PATCH 05/12] docs(changelog): match liveness wording to createdAt semantics Review feedback: the rescue keys off the newest run's createdAt, not whether a run has started. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1864780dfa..19769d07e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,8 +135,8 @@ aggregate instead: an italic *Catalog* line at the end of the version section an repo kept firing; every run that did start was green, so nothing alarmed. The cron now fires at :17 instead of :00 (GitHub documents top-of-hour scheduler overload as a delay/drop cause), and `watchdog-stuck-jobs.yml` gained a cron-liveness rescue that re-dispatches - daily-regen whenever no run has started for >10 h outside the Berlin-evening quiet window, - capping any future starvation at roughly half a day (#9649). + daily-regen whenever main has seen no new run created for >10 h outside the Berlin-evening + quiet window, capping any future starvation at roughly half a day (#9649). - **Tag search uses the GIN index and stops treating `%`/`_` as wildcards** — `SpecRepository.search_by_tags` cast the JSONB `tags` column to text and ran LIKE, which the `ix_specs_tags` GIN index can never serve (sequential scan on every MCP tag search) and which From 7e1334c34814c09b4526b66eb4833f1e2362b136 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 21:10:51 +0000 Subject: [PATCH 06/12] docs(workflows): clarify quiet window is UTC-fixed, drifts vs Berlin under CET Review feedback: the 18-21 UTC window was documented in Berlin/CEST terms, but Actions cron is UTC year-round, so the local-time mapping shifts by an hour under winter time. Document the accepted drift in both daily-regen and the watchdog liveness guard. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/daily-regen.yml | 9 +++++---- .github/workflows/watchdog-stuck-jobs.yml | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily-regen.yml b/.github/workflows/daily-regen.yml index b7c2c2d857..d8673e48fa 100644 --- a/.github/workflows/daily-regen.yml +++ b/.github/workflows/daily-regen.yml @@ -4,10 +4,11 @@ run-name: "Scheduled regen (${{ github.event.inputs.specification_id || github.e # Picks the N oldest specs (by most-recent implementation `updated` timestamp) # and re-dispatches `bulk-generate.yml` for each. Default N=1 per cron tick. # -# Schedule: every 2 hours, skipping the 20:00–23:59 Berlin (CEST) evening -# window. Berlin CEST evening hours 20, 21, 22, 23 map to UTC 18, 19, 20, 21 — -# these are intentionally skipped so runs never start during the user's -# evening. That gives 10 ticks per day. +# Schedule: every 2 hours, skipping the fixed 18:00–21:59 UTC window so runs +# never start during the user's evening. The window was sized for Berlin +# summer time (CEST, UTC+2 → 20:00–23:59 local); Actions cron is always UTC, +# so under winter time (CET, UTC+1) it covers 19:00–22:59 local instead — +# accepted drift, not a bug. That gives 10 ticks per day. # # The tick fires at minute 17 (not :00) on purpose: GitHub's scheduler is # most overloaded at the top of the hour and delays/drops `schedule` events diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index d8faae0d14..e9f058d2a7 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -305,9 +305,11 @@ jobs: # 18–21 UTC quiet window), so a >10 h silence means the schedule # is starved; re-dispatch it manually with default inputs. # Hours 17–21 UTC are skipped so the rescue never launches a - # pipeline into the Berlin-evening quiet window the cron itself + # pipeline into the fixed 18–21 UTC quiet window the cron itself # deliberately avoids (17 included because bulk-generate starts - # immediately after dispatch and would spill into it). + # immediately after dispatch and would spill into it). Like the + # cron, the window is UTC-fixed: it matches Berlin evening under + # CEST and sits an hour earlier in local terms under CET. LIVENESS_HOURS=10 HOUR=$(date -u +%-H) if (( HOUR >= 17 && HOUR <= 21 )); then From da0f86e356cc9a60c204e2379a39b62d8cc07eb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 21:15:38 +0000 Subject: [PATCH 07/12] docs: precise daily-regen tick hours and worst-case rescue latency Review feedback: 18/20 UTC are also even hours but sit in the quiet window, and the watchdog rescue's worst case is just under a day (the >10h mark can land in the skipped 17-21 UTC checks), not half a day. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- CHANGELOG.md | 3 ++- docs/workflows/overview.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19769d07e2..ef478e129c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,7 +136,8 @@ aggregate instead: an italic *Catalog* line at the end of the version section an at :17 instead of :00 (GitHub documents top-of-hour scheduler overload as a delay/drop cause), and `watchdog-stuck-jobs.yml` gained a cron-liveness rescue that re-dispatches daily-regen whenever main has seen no new run created for >10 h outside the Berlin-evening - quiet window, capping any future starvation at roughly half a day (#9649). + quiet window, capping any future starvation at typically ~half a day (worst case just under + a day, when the >10 h mark lands in the skipped 17–21 UTC checks) instead of weeks (#9649). - **Tag search uses the GIN index and stops treating `%`/`_` as wildcards** — `SpecRepository.search_by_tags` cast the JSONB `tags` column to text and ran LIKE, which the `ix_specs_tags` GIN index can never serve (sequential scan on every MCP tag search) and which diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 6f03003b52..3e9414b551 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -167,7 +167,7 @@ Located in `.github/workflows/`: | `impl-repair.yml` | Fixes rejected implementations | | `impl-merge.yml` | Merges approved PRs | | `bulk-generate.yml` | Batch implementation generation | -| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (fires at :17 past even UTC hours to dodge GitHub's top-of-hour scheduler overload) | +| `daily-regen.yml` | Cron-driven regeneration of the oldest implementations (10×/day at :17 past UTC hours 0–16 even + 22, dodging GitHub's top-of-hour scheduler overload and the 18–21 UTC quiet window) | | `watchdog-stuck-jobs.yml` | 6-hourly safety net: re-dispatches stuck reviews/repairs/merges/generations and rescues daily-regen when its cron is silently starved by GitHub (>10 h without a run) | | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main | From 1de448e14fc9d2bd5e721ee20a316293853f7ddf Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 21:19:14 +0000 Subject: [PATCH 08/12] fix(watchdog): use a fresh timestamp for the liveness gap Review feedback: NOW is captured before the A/B scans, so the gap was underestimated by however long those took. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/watchdog-stuck-jobs.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/watchdog-stuck-jobs.yml b/.github/workflows/watchdog-stuck-jobs.yml index e9f058d2a7..70549b68c3 100644 --- a/.github/workflows/watchdog-stuck-jobs.yml +++ b/.github/workflows/watchdog-stuck-jobs.yml @@ -325,7 +325,10 @@ jobs: echo "::warning::daily-regen liveness: no runs found on main — skipping rescue" else last_sec=$(date -u -d "$LAST" +%s) - gap=$(( NOW - last_sec )) + # Fresh timestamp: NOW from the top of the script is stale by + # however long the A/B scans took. + now_c=$(date -u +%s) + gap=$(( now_c - last_sec )) gap_hm=$(printf '%dh%02dm' $(( gap / 3600 )) $(( (gap % 3600) / 60 ))) if (( gap > LIVENESS_HOURS * 3600 )); then echo "::warning::daily-regen: no new run on main for ${gap_hm} (> ${LIVENESS_HOURS}h) — schedule starved, re-dispatching" From aeb656f7c44dad55c84f5a69be2e93482044ab93 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 07:51:08 +0000 Subject: [PATCH 09/12] test(impl-generate): TEMP validation pin to July-1 action + hermetic stop Pins claude-code-action to 11ba604 (last working, Jul 1) and stops the job right after the Claude step to check whether Claude produces the impl file + PNGs (permission_denials). Disables the failure handler so the run cannot comment on / retry #1010. Dispatched against this branch only via workflow_dispatch ref; main is untouched. Reverted immediately after. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/impl-generate.yml | 43 +++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index 40d4598da3..2a4a493173 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -422,7 +422,7 @@ jobs: id: claude continue-on-error: true timeout-minutes: 60 - uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 + uses: anthropics/claude-code-action@11ba60486e4aec9ddfeafcf4bb3f00b028ac2c16 # v1 (VALIDATION: July-1 pin, revert after) with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} claude_args: "--model ${{ steps.inputs.outputs.model }}" @@ -442,7 +442,7 @@ jobs: if: steps.claude.outcome == 'failure' id: claude_retry timeout-minutes: 60 - uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 + uses: anthropics/claude-code-action@11ba60486e4aec9ddfeafcf4bb3f00b028ac2c16 # v1 (VALIDATION: July-1 pin, revert after) with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} claude_args: "--model ${{ steps.inputs.outputs.model }}" @@ -458,6 +458,43 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} + # ======================================================================== + # VALIDATION TEST — inspect what Claude produced, then STOP before any + # side-effecting steps (PR / GCS / issue comment). Remove after test. + # ======================================================================== + - name: TEST — inspect Claude output and stop (validation run) + if: always() + env: + SPEC_ID: ${{ steps.inputs.outputs.specification_id }} + LANGUAGE: ${{ steps.inputs.outputs.language }} + LIBRARY: ${{ steps.inputs.outputs.library }} + EXT: ${{ steps.inputs.outputs.ext }} + run: | + echo "===== VALIDATION TEST (claude-code-action July-1 pin) =====" + IMPL_DIR="plots/${SPEC_ID}/implementations/${LANGUAGE}" + echo "--- git log (last 3) ---"; git log --oneline -3 || true + echo "--- git status --porcelain ---"; git status --porcelain || true + echo "--- ls IMPL_DIR ---"; ls -la "$IMPL_DIR" || true + for f in "${LIBRARY}${EXT}" plot-light.png plot-dark.png; do + if [ -f "$IMPL_DIR/$f" ]; then echo "PRESENT: $f"; else echo "MISSING: $f"; fi + done + echo "--- claude-execution-output.json summary ---" + python3 - <<'PY' || true + import json, glob + for p in glob.glob('/home/runner/work/_temp/claude-execution-output.json'): + try: + d = json.load(open(p)) + if isinstance(d, list): + d = d[-1] + print('is_error', d.get('is_error'), + 'num_turns', d.get('num_turns'), + 'permission_denials_count', d.get('permission_denials_count')) + except Exception as e: + print('parse err', e) + PY + echo "===== END VALIDATION TEST — stopping before side effects =====" + exit 1 + # ======================================================================== # Create metadata file (before PR) # ======================================================================== @@ -1006,7 +1043,7 @@ jobs: # Failure handling: Track failures via comments and auto-retry # ======================================================================== - name: Handle generation failure - if: failure() && steps.issue.outputs.number != '' + if: false # VALIDATION: disabled during test so it can't comment/retry on #1010 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SPEC_ID: ${{ steps.inputs.outputs.specification_id }} From 2794c7bc1cd8671d1c91e7256753047ec8d07bf1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 07:55:07 +0000 Subject: [PATCH 10/12] docs(changelog): say 'implementation regeneration' not 'spec regeneration' daily-regen re-generates implementations for the oldest specs (dispatches bulk-generate), not specs themselves; clearer wording per review. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef478e129c..e7f6f249a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,7 +129,7 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed -- **Scheduled spec regeneration no longer silently starves for days** — GitHub's scheduler +- **Scheduled implementation regeneration no longer silently starves for days** — GitHub's scheduler had been dropping `daily-regen.yml`'s cron ticks in multi-day streaks since early June (last 13 days: only 5 scheduled runs instead of ~130) while less-frequent crons in the same repo kept firing; every run that did start was green, so nothing alarmed. The cron now fires From e2d81a21d91408826b42bf9314937cee958dd653 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 20:59:38 +0000 Subject: [PATCH 11/12] test(impl-generate): reveal denied tool calls (show_full_output) Validation run refuted the action-version hypothesis: the July-1 action pin (11ba604) produced the same empty result (no PNGs, no file changes). Revert that pin to the current SHA and instead un-hide Claude's output so the next run's logs show exactly which tool calls are permission-denied. Diagnostic only; the hermetic TEST stop + disabled failure handler stay. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014vs46U1fqUJQcpf2jE3YdN --- .github/workflows/impl-generate.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index 2a4a493173..1f8512fdec 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -422,9 +422,12 @@ jobs: id: claude continue-on-error: true timeout-minutes: 60 - uses: anthropics/claude-code-action@11ba60486e4aec9ddfeafcf4bb3f00b028ac2c16 # v1 (VALIDATION: July-1 pin, revert after) + uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # VALIDATION (diagnostic only, revert after): un-hide Claude's output so + # the logs reveal exactly which tool calls are being permission-denied. + show_full_output: true claude_args: "--model ${{ steps.inputs.outputs.model }}" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -442,9 +445,12 @@ jobs: if: steps.claude.outcome == 'failure' id: claude_retry timeout-minutes: 60 - uses: anthropics/claude-code-action@11ba60486e4aec9ddfeafcf4bb3f00b028ac2c16 # v1 (VALIDATION: July-1 pin, revert after) + uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # VALIDATION (diagnostic only, revert after): un-hide Claude's output so + # the logs reveal exactly which tool calls are being permission-denied. + show_full_output: true claude_args: "--model ${{ steps.inputs.outputs.model }}" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' From 5bf1678ae3f6e7ee00e3ff00f4c3d3c92ad0a585 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:10:37 +0200 Subject: [PATCH 12/12] revert: drop all temporary validation changes to impl-generate.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the pipeline breakage is found and fixed in PR #9651 — no further validation runs on this branch are needed or even possible: the action's OAuth token exchange 401s any run whose workflow file differs from the default branch ('Workflow validation failed. The workflow file must exist and have identical content to the version on the repository's default branch.'). That 401 is also why the earlier July-1-pin validation run produced an empty result: Claude never started, so it neither confirmed nor refuted the action-version hypothesis. The actual mechanism (confirmed by local repro): CLI 2.1.171+ ignores permissions.allow from .claude/settings.json in untrusted workspaces, and CI checkouts are never trusted; explicit --settings in claude_args is honored and fixes it. impl-generate.yml is now byte-identical to main again, leaving this PR with only its cron-offset + watchdog-liveness content. Co-Authored-By: Claude Fable 5 --- .github/workflows/impl-generate.yml | 45 +---------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml index 1f8512fdec..40d4598da3 100644 --- a/.github/workflows/impl-generate.yml +++ b/.github/workflows/impl-generate.yml @@ -425,9 +425,6 @@ jobs: uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - # VALIDATION (diagnostic only, revert after): un-hide Claude's output so - # the logs reveal exactly which tool calls are being permission-denied. - show_full_output: true claude_args: "--model ${{ steps.inputs.outputs.model }}" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -448,9 +445,6 @@ jobs: uses: anthropics/claude-code-action@1298632ce7736903d02a1435002705aa2a594a6c # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - # VALIDATION (diagnostic only, revert after): un-hide Claude's output so - # the logs reveal exactly which tool calls are being permission-denied. - show_full_output: true claude_args: "--model ${{ steps.inputs.outputs.model }}" # bulk-generate dispatches us from the github-actions bot; explicitly allow it. allowed_bots: '*' @@ -464,43 +458,6 @@ jobs: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - IS_REGENERATION: ${{ steps.existing.outputs.is_regeneration }} - # ======================================================================== - # VALIDATION TEST — inspect what Claude produced, then STOP before any - # side-effecting steps (PR / GCS / issue comment). Remove after test. - # ======================================================================== - - name: TEST — inspect Claude output and stop (validation run) - if: always() - env: - SPEC_ID: ${{ steps.inputs.outputs.specification_id }} - LANGUAGE: ${{ steps.inputs.outputs.language }} - LIBRARY: ${{ steps.inputs.outputs.library }} - EXT: ${{ steps.inputs.outputs.ext }} - run: | - echo "===== VALIDATION TEST (claude-code-action July-1 pin) =====" - IMPL_DIR="plots/${SPEC_ID}/implementations/${LANGUAGE}" - echo "--- git log (last 3) ---"; git log --oneline -3 || true - echo "--- git status --porcelain ---"; git status --porcelain || true - echo "--- ls IMPL_DIR ---"; ls -la "$IMPL_DIR" || true - for f in "${LIBRARY}${EXT}" plot-light.png plot-dark.png; do - if [ -f "$IMPL_DIR/$f" ]; then echo "PRESENT: $f"; else echo "MISSING: $f"; fi - done - echo "--- claude-execution-output.json summary ---" - python3 - <<'PY' || true - import json, glob - for p in glob.glob('/home/runner/work/_temp/claude-execution-output.json'): - try: - d = json.load(open(p)) - if isinstance(d, list): - d = d[-1] - print('is_error', d.get('is_error'), - 'num_turns', d.get('num_turns'), - 'permission_denials_count', d.get('permission_denials_count')) - except Exception as e: - print('parse err', e) - PY - echo "===== END VALIDATION TEST — stopping before side effects =====" - exit 1 - # ======================================================================== # Create metadata file (before PR) # ======================================================================== @@ -1049,7 +1006,7 @@ jobs: # Failure handling: Track failures via comments and auto-retry # ======================================================================== - name: Handle generation failure - if: false # VALIDATION: disabled during test so it can't comment/retry on #1010 + if: failure() && steps.issue.outputs.number != '' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SPEC_ID: ${{ steps.inputs.outputs.specification_id }}