From 8846f6959d1fcebba8806e5e126543f580d1d08e Mon Sep 17 00:00:00 2001
From: Justin Middler
Date: Thu, 30 Jul 2026 15:29:51 +1000
Subject: [PATCH 1/2] fix(web): let operators close expired approvals and retry
stuck agent runs
Two dead ends where the only escape was the database.
Approvals: decide() refused BOTH verdicts once expiresAt passed, and
nothing ever moved a row out of pending, so an expired request sat in the
inbox forever offering an Approve button that could never succeed.
- Approving past the deadline stays blocked. That is what expiry is for.
- Rejecting is now allowed: it is strictly de-escalating and it is the
only way to close the row with a recorded reason.
- expireOverdue() transitions overdue rows to the 'expired' status the
enum already had, with an audit event, lazily on inbox read so a
workspace self-heals without a scheduler.
- Command stops counting overdue rows as pending approvals.
- The inbox drops Approve once overdue and offers 'Reject and close',
instead of rendering live buttons above 'Approval has expired.'
Agent runs: re-dispatching a failed task already worked, but a run wedged
at queued/running/awaiting_approval blocked the button permanently and
POST /tasks/[id]/cancel was wired nowhere in the portal.
- Cancel run appears whenever a run is in flight.
- A settled failure now reads 'Retry dispatch' and says what happened.
- The blocked message names the way out rather than just refusing.
Co-Authored-By: Claude Opus 5 (1M context)
---
apps/web/app/api/v1/approvals/route.ts | 5 +-
.../features/approvals/governance-inbox.tsx | 38 +++++----
.../operations/operations-view.test.ts | 28 ++++++-
.../features/operations/operations-view.tsx | 82 +++++++++++++++----
apps/web/lib/approval-expiry.test.ts | 55 +++++++++++++
apps/web/lib/command-summary-domain.ts | 5 +-
apps/web/lib/integration-action-domain.ts | 58 ++++++++++++-
apps/web/lib/queries/hooks.ts | 25 ++++++
8 files changed, 261 insertions(+), 35 deletions(-)
create mode 100644 apps/web/lib/approval-expiry.test.ts
diff --git a/apps/web/app/api/v1/approvals/route.ts b/apps/web/app/api/v1/approvals/route.ts
index 91e8824..e3bf84f 100644
--- a/apps/web/app/api/v1/approvals/route.ts
+++ b/apps/web/app/api/v1/approvals/route.ts
@@ -5,7 +5,10 @@ export async function GET(request: Request) {
const traceId = requestTraceId(request);
try {
return Response.json({
- data: await new ApprovalDomainService().list(await apiSubject(request)),
+ data: await new ApprovalDomainService().list(
+ await apiSubject(request),
+ traceId,
+ ),
traceId,
});
} catch (error) {
diff --git a/apps/web/features/approvals/governance-inbox.tsx b/apps/web/features/approvals/governance-inbox.tsx
index 8d0fd92..efaf6de 100644
--- a/apps/web/features/approvals/governance-inbox.tsx
+++ b/apps/web/features/approvals/governance-inbox.tsx
@@ -209,7 +209,11 @@ function ApprovalDetail({
}) {
const severity = riskSeverity(approval.riskSummary);
const highImpact = severity === "critical";
- const pending = approval.status === "pending";
+ // A row can still read as pending until the next inbox load expires it, so
+ // trust the deadline rather than the stored status for what is offerable.
+ const overdue = new Date(approval.expiresAt) <= new Date();
+ const pending = approval.status === "pending" && !overdue;
+ const closable = approval.status === "pending" && overdue;
return (
@@ -277,9 +281,15 @@ function ApprovalDetail({
- {pending ? (
+ {pending || closable ? (
-
) : (
- This approval is no longer pending.
+ {approval.status === "expired"
+ ? "This approval expired without a decision. Nothing was executed."
+ : "This approval is no longer pending."}
{approval.reason ? ` Reason: ${approval.reason}` : ""}
)}
diff --git a/apps/web/features/operations/operations-view.test.ts b/apps/web/features/operations/operations-view.test.ts
index 9441069..aa1da75 100644
--- a/apps/web/features/operations/operations-view.test.ts
+++ b/apps/web/features/operations/operations-view.test.ts
@@ -29,7 +29,7 @@ describe("Operations board", () => {
expect(view).toContain("dispatchBlockedReason");
// Every refusal path returns operator-readable text.
expect(view).toContain("Assign this task to an agent to dispatch it.");
- expect(view).toContain("already has an active agent run");
+ expect(view).toContain("A run is already in flight.");
expect(view).toContain("assigneeReadinessReason");
});
@@ -103,3 +103,29 @@ describe("Task composer", () => {
expect(composer).not.toMatch(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-/);
});
});
+
+describe("Stuck and failed agent work", () => {
+ it("offers cancel while a run is in flight", async () => {
+ const view = await source("./operations-view.tsx");
+ expect(view).toContain("useCancelTaskRun");
+ expect(view).toContain("Cancel run");
+ // Every status the server treats as in-flight must be escapable, not just
+ // queued/running — awaiting_approval wedges a task just as hard.
+ expect(view).toContain('"awaiting_approval"');
+ expect(view).toContain('"waiting_sources"');
+ });
+
+ it("labels a re-dispatch as a retry and says why", async () => {
+ const view = await source("./operations-view.tsx");
+ expect(view).toContain("function isRetry");
+ expect(view).toContain("Retry dispatch");
+ expect(view).toContain("Previous run");
+ });
+
+ it("points a blocked dispatch at the way out", async () => {
+ const view = await source("./operations-view.tsx");
+ expect(view).toContain(
+ "A run is already in flight. Cancel it before dispatching again.",
+ );
+ });
+});
diff --git a/apps/web/features/operations/operations-view.tsx b/apps/web/features/operations/operations-view.tsx
index 5bdc61a..bb52b07 100644
--- a/apps/web/features/operations/operations-view.tsx
+++ b/apps/web/features/operations/operations-view.tsx
@@ -26,6 +26,7 @@ import {
type ComposerSeed,
} from "@/features/operations/task-composer";
import {
+ useCancelTaskRun,
useDelegateTask,
useTasks,
useUpdateTask,
@@ -162,12 +163,24 @@ function taskToBoardItem(task: RawTask): BoardItem {
};
}
-/** A run that is already queued or running must not be dispatched again. */
+/** Statuses the server treats as an in-flight run; cancel is the only exit. */
+const ACTIVE_RUN_STATUSES = [
+ "queued",
+ "running",
+ "awaiting_approval",
+ "waiting_sources",
+];
+
+function hasActiveRun(item: BoardItem): boolean {
+ return ACTIVE_RUN_STATUSES.includes(item.agentRunStatus ?? "");
+}
+
+/** A run already in flight must not be dispatched again. */
function dispatchBlockedReason(item: BoardItem): string | null {
if (!item.assigneeIsAgent)
return "Assign this task to an agent to dispatch it.";
- if (item.agentRunStatus === "queued" || item.agentRunStatus === "running")
- return "This task already has an active agent run.";
+ if (hasActiveRun(item))
+ return "A run is already in flight. Cancel it before dispatching again.";
if (item.assigneeReadiness !== "ready")
return (
item.assigneeReadinessReason ?? "Assigned agent is not ready for work."
@@ -175,6 +188,13 @@ function dispatchBlockedReason(item: BoardItem): string | null {
return null;
}
+/** A settled run can be handed back to the agent; label it as a retry. */
+function isRetry(item: BoardItem): boolean {
+ return (
+ item.agentRunStatus === "failed" || item.agentRunStatus === "cancelled"
+ );
+}
+
export function OperationsView() {
const tasks = useTasks();
const updateTask = useUpdateTask();
@@ -473,6 +493,7 @@ export function OperationsView() {
function DetailDrawer({ item }: { item: BoardItem | null }) {
const delegateTask = useDelegateTask();
+ const cancelRun = useCancelTaskRun();
const [error, setError] = useState(null);
const [notice, setNotice] = useState(null);
@@ -501,6 +522,20 @@ function DetailDrawer({ item }: { item: BoardItem | null }) {
}
}
+ async function cancel() {
+ if (!item) return;
+ setError(null);
+ setNotice(null);
+ try {
+ await cancelRun.mutateAsync(item.id);
+ setNotice("Run cancelled. You can dispatch it again.");
+ } catch (caught) {
+ setError(
+ caught instanceof Error ? caught.message : "Could not cancel the run.",
+ );
+ }
+ }
+
return (
-
+
+ {hasActiveRun(item) ? (
+
+ ) : null}
+
+
-
+
{blocked ??
- "Runs under the agent's governed capability envelope. External writes stay approval-gated."}
+ (isRetry(item)
+ ? `Previous run ${item.agentRunStatus}. Dispatching again starts a fresh run under the agent's governed capability envelope.`
+ : "Runs under the agent's governed capability envelope. External writes stay approval-gated.")}
{error ? (
diff --git a/apps/web/lib/approval-expiry.test.ts b/apps/web/lib/approval-expiry.test.ts
new file mode 100644
index 0000000..d3d58de
--- /dev/null
+++ b/apps/web/lib/approval-expiry.test.ts
@@ -0,0 +1,55 @@
+import { readFile } from "node:fs/promises";
+import { describe, expect, it } from "vitest";
+
+async function source(name: string) {
+ return readFile(new URL(name, import.meta.url), "utf8");
+}
+
+/**
+ * An expired approval used to be a permanent dead row: `decide` refused both
+ * verdicts, and nothing ever moved it out of `pending`.
+ */
+describe("approval expiry", () => {
+ it("refuses approval past the deadline but allows rejection", async () => {
+ const domain = await source("./integration-action-domain.ts");
+ expect(domain).toContain(
+ 'approval.expiresAt <= new Date() && decision.status !== "rejected"',
+ );
+ expect(domain).toContain("Reject it to close it out.");
+ });
+
+ it("transitions overdue rows to expired with an audit event", async () => {
+ const domain = await source("./integration-action-domain.ts");
+ expect(domain).toContain("async expireOverdue(");
+ expect(domain).toContain('set({ status: "expired"');
+ expect(domain).toContain("lte(schema.approvals.expiresAt, new Date())");
+ expect(domain).toContain('action: "workflow.approval.expired"');
+ // Expiry must run before the inbox is read, or the UI keeps offering
+ // Approve on a request that can no longer be approved.
+ expect(domain).toContain("await this.expireOverdue(subject.organisationId");
+ });
+
+ it("never counts an overdue approval as actionable on Command", async () => {
+ const summary = await source("./command-summary-domain.ts");
+ expect(summary).toContain("gt(schema.approvals.expiresAt, new Date())");
+ });
+});
+
+describe("approval inbox controls", () => {
+ it("drops Approve and offers a closing Reject once overdue", async () => {
+ const view = await source("../features/approvals/governance-inbox.tsx");
+ expect(view).toContain("const overdue = new Date(approval.expiresAt)");
+ expect(view).toContain(
+ 'const closable = approval.status === "pending" && overdue',
+ );
+ expect(view).toContain("{closable ? null : (");
+ expect(view).toContain('{closable ? "Reject and close" : "Reject"}');
+ expect(view).toContain("can no longer be");
+ });
+
+ it("explains an expired outcome rather than saying only 'not pending'", async () => {
+ const view = await source("../features/approvals/governance-inbox.tsx");
+ expect(view).toContain('approval.status === "expired"');
+ expect(view).toContain("Nothing was executed.");
+ });
+});
diff --git a/apps/web/lib/command-summary-domain.ts b/apps/web/lib/command-summary-domain.ts
index 96a0348..7d276b6 100644
--- a/apps/web/lib/command-summary-domain.ts
+++ b/apps/web/lib/command-summary-domain.ts
@@ -1,4 +1,4 @@
-import { and, desc, eq, inArray, isNull } from "drizzle-orm";
+import { and, desc, eq, gt, inArray, isNull } from "drizzle-orm";
import {
hasCapability,
type AuthorisationSubject,
@@ -74,6 +74,9 @@ export async function getCommandSummary(
and(
eq(schema.approvals.organisationId, subject.organisationId),
eq(schema.approvals.status, "pending"),
+ // An overdue row is still stored as pending until something reads
+ // the inbox and expires it. Never count it as actionable here.
+ gt(schema.approvals.expiresAt, new Date()),
),
)
.orderBy(desc(schema.approvals.requestedAt))
diff --git a/apps/web/lib/integration-action-domain.ts b/apps/web/lib/integration-action-domain.ts
index 805f438..6413fcc 100644
--- a/apps/web/lib/integration-action-domain.ts
+++ b/apps/web/lib/integration-action-domain.ts
@@ -18,7 +18,7 @@ import {
IntegrationActionRequestSchema,
type IntegrationActionRequest,
} from "@muster/integrations";
-import { and, desc, eq, inArray } from "drizzle-orm";
+import { and, desc, eq, inArray, lte } from "drizzle-orm";
import { z } from "zod";
import { ApiProblem } from "./api-context.ts";
@@ -348,8 +348,50 @@ export class IntegrationActionDomainService {
export class ApprovalDomainService {
constructor(private readonly db = database()) {}
- async list(subject: AuthorisationSubject) {
+ /**
+ * Move overdue pending approvals to `expired` before anyone reads them.
+ *
+ * Nothing else transitions them, so without this they stay `pending`
+ * forever: the inbox keeps offering Approve on a request that can no longer
+ * be approved, and the attention queue keeps counting dead rows. Lazy
+ * expiry runs on read so a workspace self-heals without a scheduler.
+ */
+ async expireOverdue(organisationId: string, traceId: string) {
+ return this.db.transaction(async (tx) => {
+ const overdue = await tx
+ .update(schema.approvals)
+ .set({ status: "expired", decisionAt: new Date() })
+ .where(
+ and(
+ eq(schema.approvals.organisationId, organisationId),
+ eq(schema.approvals.status, "pending"),
+ lte(schema.approvals.expiresAt, new Date()),
+ ),
+ )
+ .returning({
+ id: schema.approvals.id,
+ actionType: schema.approvals.actionType,
+ requestingActorId: schema.approvals.requestingActorId,
+ });
+ for (const approval of overdue) {
+ await appendAuditEvent(tx, {
+ organisationId,
+ actorId: approval.requestingActorId,
+ actorType: "system",
+ action: "workflow.approval.expired",
+ targetType: "approval",
+ targetId: approval.id,
+ metadata: { actionType: approval.actionType },
+ traceId,
+ });
+ }
+ return overdue.length;
+ });
+ }
+
+ async list(subject: AuthorisationSubject, traceId = "approval-list") {
requireCapability(subject, "workflows.approve");
+ await this.expireOverdue(subject.organisationId, traceId);
const rows = await this.db
.select()
.from(schema.approvals)
@@ -387,8 +429,16 @@ export class ApprovalDomainService {
);
if (approval.status !== "pending")
return { id: approval.id, status: approval.status, duplicate: true };
- if (approval.expiresAt <= new Date())
- throw new ApiProblem(409, "Approval expired", "Approval has expired.");
+ // Approving an expired dangerous action is exactly what expiry exists to
+ // prevent. Rejecting one is strictly de-escalating, so it stays open —
+ // otherwise the row can never be closed and clutters the queue forever.
+ if (approval.expiresAt <= new Date() && decision.status !== "rejected") {
+ throw new ApiProblem(
+ 409,
+ "Approval expired",
+ "This approval expired and can no longer be approved. Reject it to close it out.",
+ );
+ }
if (!capabilities.includes(approval.requiredCapability as Capability))
throw new Error("Approval requires an unknown capability");
requireCapability(subject, approval.requiredCapability as Capability);
diff --git a/apps/web/lib/queries/hooks.ts b/apps/web/lib/queries/hooks.ts
index c71a81f..d66d557 100644
--- a/apps/web/lib/queries/hooks.ts
+++ b/apps/web/lib/queries/hooks.ts
@@ -336,6 +336,31 @@ export function useCreateTask() {
});
}
+/**
+ * Cancel a task's in-flight agent run. Without this a run wedged at
+ * queued/running (gateway crash, expired lease) blocks re-dispatch forever
+ * with no operator escape.
+ */
+export function useCancelTaskRun() {
+ const client = useQueryClient();
+ return useMutation({
+ mutationFn: async (taskId: string) => {
+ const res = await apiPost<{ status?: string }>(
+ `/api/v1/tasks/${taskId}/cancel`,
+ {},
+ );
+ return res.data;
+ },
+ onSuccess: async () => {
+ await Promise.all([
+ client.invalidateQueries({ queryKey: queryKeys.tasks }),
+ client.invalidateQueries({ queryKey: queryKeys.commandSummary }),
+ client.invalidateQueries({ queryKey: ["audit"] }),
+ ]);
+ },
+ });
+}
+
/**
* Hand a task to its assigned agent. The server re-checks capability,
* readiness, and kill switch — this only asks.
From b54e64029a823b00c049d4ab0cd726184fe7fcdb Mon Sep 17 00:00:00 2001
From: Justin Middler
Date: Thu, 30 Jul 2026 15:45:02 +1000
Subject: [PATCH 2/2] fix(web): keep expired approvals rejectable after the
lazy sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Self-inflicted by the previous commit and caught in review.
expireOverdue() runs on every inbox list, so a row becomes 'expired'
the moment anyone opens Approvals. decide() then hit its
'status !== pending' early return and silently answered duplicate:true,
and the inbox only treated overdue *pending* rows as closable — so the
act of opening the page removed the only way to close the row.
Rejection now survives the expired state on both sides.
Co-Authored-By: Claude Opus 5 (1M context)
---
.../features/approvals/governance-inbox.tsx | 5 +++-
apps/web/lib/approval-expiry.test.ts | 27 ++++++++++++++++---
apps/web/lib/integration-action-domain.ts | 7 ++++-
3 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/apps/web/features/approvals/governance-inbox.tsx b/apps/web/features/approvals/governance-inbox.tsx
index efaf6de..8b2c505 100644
--- a/apps/web/features/approvals/governance-inbox.tsx
+++ b/apps/web/features/approvals/governance-inbox.tsx
@@ -213,7 +213,10 @@ function ApprovalDetail({
// trust the deadline rather than the stored status for what is offerable.
const overdue = new Date(approval.expiresAt) <= new Date();
const pending = approval.status === "pending" && !overdue;
- const closable = approval.status === "pending" && overdue;
+ // Includes rows already stored as `expired` by the inbox's lazy sweep, not
+ // just ones that are still nominally pending.
+ const closable =
+ approval.status === "expired" || (approval.status === "pending" && overdue);
return (
diff --git a/apps/web/lib/approval-expiry.test.ts b/apps/web/lib/approval-expiry.test.ts
index d3d58de..9053166 100644
--- a/apps/web/lib/approval-expiry.test.ts
+++ b/apps/web/lib/approval-expiry.test.ts
@@ -39,9 +39,7 @@ describe("approval inbox controls", () => {
it("drops Approve and offers a closing Reject once overdue", async () => {
const view = await source("../features/approvals/governance-inbox.tsx");
expect(view).toContain("const overdue = new Date(approval.expiresAt)");
- expect(view).toContain(
- 'const closable = approval.status === "pending" && overdue',
- );
+ expect(view).toContain('(approval.status === "pending" && overdue)');
expect(view).toContain("{closable ? null : (");
expect(view).toContain('{closable ? "Reject and close" : "Reject"}');
expect(view).toContain("can no longer be");
@@ -53,3 +51,26 @@ describe("approval inbox controls", () => {
expect(view).toContain("Nothing was executed.");
});
});
+
+/**
+ * The lazy sweep and the decision path have to agree. If listing the inbox
+ * moves a row to `expired` and `decide` then treats `expired` as terminal,
+ * opening Approvals is what destroys the only way to close the row.
+ */
+describe("expired rows stay closable after the lazy sweep", () => {
+ it("does not short-circuit a rejection on a stored expired row", async () => {
+ const domain = await source("./integration-action-domain.ts");
+ expect(domain).toContain("const closingExpired =");
+ expect(domain).toContain(
+ 'approval.status === "expired" && decision.status === "rejected"',
+ );
+ expect(domain).toContain(
+ 'if (approval.status !== "pending" && !closingExpired)',
+ );
+ });
+
+ it("still offers the closing control once the row reads expired", async () => {
+ const view = await source("../features/approvals/governance-inbox.tsx");
+ expect(view).toContain('approval.status === "expired" ||');
+ });
+});
diff --git a/apps/web/lib/integration-action-domain.ts b/apps/web/lib/integration-action-domain.ts
index 6413fcc..2a54535 100644
--- a/apps/web/lib/integration-action-domain.ts
+++ b/apps/web/lib/integration-action-domain.ts
@@ -427,7 +427,12 @@ export class ApprovalDomainService {
"Approval not found",
"Approval does not exist.",
);
- if (approval.status !== "pending")
+ // A row lands on `expired` as soon as anything lists the inbox, so
+ // rejection has to survive that state too — otherwise the very act of
+ // opening Approvals removes the only way to close the row.
+ const closingExpired =
+ approval.status === "expired" && decision.status === "rejected";
+ if (approval.status !== "pending" && !closingExpired)
return { id: approval.id, status: approval.status, duplicate: true };
// Approving an expired dangerous action is exactly what expiry exists to
// prevent. Rejecting one is strictly de-escalating, so it stays open —