From 7afcc3262b144c46dc7cec3dfa88be5b68899901 Mon Sep 17 00:00:00 2001
From: howenyap <wenhyap02@gmail.com>
Date: Wed, 15 Jul 2026 17:08:52 +0800
Subject: [PATCH 1/2] feat(eval): propagate trace source framework

---
 hud/cli/eval.py                         | 19 +++++-
 hud/cli/tests/test_eval_config.py       | 17 ++++++
 hud/environment/tests/test_legacy.py    |  8 ++-
 hud/eval/__init__.py                    |  2 +
 hud/eval/chat.py                        | 19 +++++-
 hud/eval/job.py                         | 10 +++-
 hud/eval/run.py                         | 17 +++++-
 hud/eval/runtime.py                     | 31 +++++++++-
 hud/eval/source_framework.py            | 27 +++++++++
 hud/eval/task.py                        |  5 ++
 hud/eval/taskset.py                     | 14 ++++-
 hud/eval/tests/test_chat.py             | 25 ++++++--
 hud/eval/tests/test_hosted.py           | 80 +++++++++++++++++++++----
 hud/eval/tests/test_job.py              | 10 ++++
 hud/eval/tests/test_local_runtime.py    | 21 ++++---
 hud/eval/tests/test_rollout.py          | 57 ++++++++++++++----
 hud/eval/tests/test_source_framework.py | 25 ++++++++
 hud/eval/tests/test_task.py             |  2 +-
 18 files changed, 343 insertions(+), 46 deletions(-)
 create mode 100644 hud/eval/source_framework.py
 create mode 100644 hud/eval/tests/test_source_framework.py

diff --git a/hud/cli/eval.py b/hud/cli/eval.py
index 0a071b53b..b17a38dff 100644
--- a/hud/cli/eval.py
+++ b/hud/cli/eval.py
@@ -740,7 +740,13 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
 
     if cfg.remote:
         require_api_key("run remote hosted evals")
-        return HostedRuntime()
+        # A platform taskset has no local source tree. All currently exported
+        # platform tasksets are HUD-authored; future export records can carry
+        # their framework explicitly here.
+        return HostedRuntime(
+            source=source_path,
+            source_framework="hud" if source_path is None else None,
+        )
     if cfg.runtime == "local":
         if source_path is None:
             raise ValueError("local placement requires a local source path")
@@ -833,12 +839,23 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
 
     agent = _build_agent(cfg)
     placement = _resolve_placement(cfg, source_path if is_local else None)
+    if is_local:
+        from hud.eval import resolve_source_framework
+
+        source_root = source_path if source_path.is_dir() else source_path.parent
+        source_framework = resolve_source_framework(source_root)
+        if source_framework is None:
+            raise ValueError(f"could not determine source framework from {source_root}")
+    else:
+        # Platform tasksets currently contain HUD-authored environments.
+        source_framework = "hud"
 
     job = await taskset.run(
         agent,
         runtime=placement,
         group=cfg.group_size,
         max_concurrent=cfg.max_concurrent,
+        source_framework=source_framework,
     )
     if job.runs and settings.telemetry_enabled and settings.api_key:
         hud_console.info(f"{settings.hud_web_url}/jobs/{job.id}")
diff --git a/hud/cli/tests/test_eval_config.py b/hud/cli/tests/test_eval_config.py
index 6b94f0b23..cf7ac5c11 100644
--- a/hud/cli/tests/test_eval_config.py
+++ b/hud/cli/tests/test_eval_config.py
@@ -148,6 +148,23 @@ def test_resolve_placement_remote_uses_hosted_runtime(
     placement = eval_mod._resolve_placement(EvalConfig(remote=True), tmp_path)
 
     assert isinstance(placement, HostedRuntime)
+    assert placement.source == tmp_path
+    assert placement.source_framework is None
+
+
+def test_resolve_placement_remote_platform_taskset_declares_hud_framework(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from hud.eval import HostedRuntime
+    from hud.settings import settings
+
+    monkeypatch.setattr(settings, "api_key", "sk-hud-test")
+
+    placement = eval_mod._resolve_placement(EvalConfig(remote=True), None)
+
+    assert isinstance(placement, HostedRuntime)
+    assert placement.source is None
+    assert placement.source_framework == "hud"
 
 
 def test_runtime_cli_override_clears_config_remote() -> None:
diff --git a/hud/environment/tests/test_legacy.py b/hud/environment/tests/test_legacy.py
index 8234dae46..8d8c6cc44 100644
--- a/hud/environment/tests/test_legacy.py
+++ b/hud/environment/tests/test_legacy.py
@@ -120,7 +120,11 @@ async def test_taskset_concurrent_grouped_rollouts() -> None:
     taskset = Taskset("adds", (add(a=i, b=i + 1) for i in range(4)))
 
     job = await taskset.run(
-        _FnAgent(_solve_add), runtime=lambda _row: _local(env), group=2, max_concurrent=3
+        _FnAgent(_solve_add),
+        runtime=lambda _row: _local(env),
+        group=2,
+        max_concurrent=3,
+        source_framework="hud",
     )
     runs = job.runs
 
@@ -144,7 +148,7 @@ def solve_or_boom(prompt: str) -> str:
         return _solve_add(prompt)
 
     job = await Taskset("adds", (add(a=i, b=1) for i in range(4))).run(
-        _FnAgent(solve_or_boom), runtime=lambda _row: _local(env)
+        _FnAgent(solve_or_boom), runtime=lambda _row: _local(env), source_framework="hud"
     )
     runs = job.runs
 
diff --git a/hud/eval/__init__.py b/hud/eval/__init__.py
index 2824645c7..696139dd8 100644
--- a/hud/eval/__init__.py
+++ b/hud/eval/__init__.py
@@ -48,6 +48,7 @@
     RuntimeResources,
     SubprocessRuntime,
 )
+from .source_framework import resolve_source_framework
 from .sync import SyncPlan
 from .task import Task
 from .taskset import Taskset
@@ -74,5 +75,6 @@
     "Task",
     "Taskset",
     "Trace",
+    "resolve_source_framework",
     "rollout",
 ]
diff --git a/hud/eval/chat.py b/hud/eval/chat.py
index 17fee982d..92c86d775 100644
--- a/hud/eval/chat.py
+++ b/hud/eval/chat.py
@@ -11,7 +11,11 @@
     from hud.agents import create_agent
     from tasks import assistant  # an @env.template taking ``messages``
 
-    chat = Chat(assistant(messages=[]), create_agent("claude-sonnet-4-5"))
+    chat = Chat(
+        assistant(messages=[]),
+        create_agent("claude-sonnet-4-5"),
+        source_framework="hud",
+    )
     r1 = await chat.send("Book me a flight")
     r2 = await chat.send("SFO to JFK")
 
@@ -38,6 +42,7 @@
     from hud.agents.base import Agent
 
     from .runtime import Provider
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 LOGGER = logging.getLogger(__name__)
@@ -84,6 +89,7 @@ def __init__(
         /,
         *,
         runtime: Provider | None = None,
+        source_framework: SourceFrameworkName = "hud",
     ) -> None:
         """Initialize Chat.
 
@@ -99,10 +105,13 @@ def __init__(
                 ``LocalRuntime("env.py")`` or ``Runtime("tcp://...")``. Chat is
                 interactive and local: it drives the agent loop in this process,
                 so hosted placement does not apply.
+            source_framework: Explicit provenance for a served runtime whose
+                local source tree is unavailable.
         """
         self._task = task
         self._agent = agent
         self._runtime = runtime
+        self._source_framework: SourceFrameworkName = source_framework
         self.messages: list[dict[str, Any]] = []
         #: The conversation's job — every turn's run reports under it
         #: (started on the first ``send``).
@@ -138,7 +147,13 @@ async def send(self, message: MessageContent) -> Trace:
             )
         if self.job is None:  # one job spans the whole conversation
             self.job = await Job.start(self._task.id)
-        run = await rollout(task, self._agent, runtime=self._runtime, job_id=self.job.id)
+        run = await rollout(
+            task,
+            self._agent,
+            runtime=self._runtime,
+            job_id=self.job.id,
+            source_framework=self._source_framework,
+        )
         self.job.runs.append(run)
         result = run.trace
         if result.is_error:
diff --git a/hud/eval/job.py b/hud/eval/job.py
index 985eb2d96..d3fc9f67d 100644
--- a/hud/eval/job.py
+++ b/hud/eval/job.py
@@ -105,18 +105,26 @@ async def trace_enter(
     *,
     job_id: str | None,
     group_id: str | None,
+    source_framework: str = "hud",
     model: str | None = None,
 ) -> None:
     """Report that one rollout started.
 
     ``model`` is the model string the agent will sample (when known); the
     platform resolves it and attributes the trace immediately on enter.
+    Resolve ``source_framework`` from the environment source tree before
+    entering the trace.
     """
     if not _reporting_enabled():
         return
     await _report(
         f"/trace/{trace_id}/enter",
-        {"job_id": job_id, "group_id": group_id, "model": model},
+        {
+            "job_id": job_id,
+            "group_id": group_id,
+            "model": model,
+            "source_framework": source_framework,
+        },
     )
 
 
diff --git a/hud/eval/run.py b/hud/eval/run.py
index 5949786bd..82aea0ee5 100644
--- a/hud/eval/run.py
+++ b/hud/eval/run.py
@@ -5,7 +5,12 @@
 loopback, a container, a cloud sandbox), starts the task, drives the agent,
 grades, and tears down, filling a :class:`Run` along the way::
 
-    run = await rollout(task, agent, runtime=LocalRuntime("env.py"))
+    run = await rollout(
+        task,
+        agent,
+        runtime=LocalRuntime("env.py"),
+        source_framework="hud",
+    )
 
 It is the *client-here* path: the agent loop runs in this process against a
 :class:`~hud.eval.runtime.Provider`'s channel. The same driver runs on the
@@ -45,6 +50,7 @@
     from hud.clients.client import HudClient
 
     from .runtime import Provider, Runtime
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 logger = logging.getLogger("hud.eval.run")
@@ -283,6 +289,7 @@ async def rollout(
     group_id: str | None = None,
     trace_id: str | None = None,
     rollout_timeout: float | None = None,
+    source_framework: SourceFrameworkName = "hud",
 ) -> Run:
     """Drive one task to a graded :class:`Run` here, against ``runtime``'s channel.
 
@@ -321,7 +328,13 @@ async def rollout(
 
     agent_model = agent.config.model if isinstance(agent, ToolAgent) else None
     with set_trace_context(trace_id):
-        await trace_enter(trace_id, job_id=job_id, group_id=group_id, model=agent_model)
+        await trace_enter(
+            trace_id,
+            job_id=job_id,
+            group_id=group_id,
+            model=agent_model,
+            source_framework=source_framework,
+        )
         run: Run | None = None
         _phase = "provisioning"
 
diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py
index a05e16ab1..5579f420b 100644
--- a/hud/eval/runtime.py
+++ b/hud/eval/runtime.py
@@ -58,6 +58,7 @@
     from hud.agents.base import Agent
     from hud.environment.env import Environment
 
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 logger = logging.getLogger("hud.eval.runtime")
@@ -898,6 +899,7 @@ class HUDRuntime:
     def __init__(self, *, run_timeout: float = 3600.0, runtime_url: str | None = None) -> None:
         self.run_timeout = run_timeout
         self.runtime_url = runtime_url
+        self.source_framework: SourceFrameworkName = "hud"
 
     async def run(
         self,
@@ -915,6 +917,7 @@ async def run(
             trace_id=trace_id,
             job_id=job_id,
             group_id=group_id,
+            source_framework=self.source_framework,
         )
 
     def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]:
@@ -1034,9 +1037,17 @@ class HostedRuntime:
     def __init__(
         self,
         *,
+        source: str | Path | None = None,
+        source_framework: SourceFrameworkName | None = None,
         poll_interval: float = 5.0,
         run_timeout: float = 3600.0,
     ) -> None:
+        # Directory used to resolve ``source_framework`` (expects Dockerfile.hud).
+        self.source: Path | None = None
+        if source is not None:
+            path = Path(source).expanduser().resolve()
+            self.source = path if path.is_dir() else path.parent
+        self.source_framework: SourceFrameworkName | None = source_framework
         self.poll_interval = poll_interval
         self.run_timeout = run_timeout
 
@@ -1048,6 +1059,7 @@ async def run(
         job_id: str,
         group_id: str | None = None,
         trace_id: str | None = None,
+        source_framework: SourceFrameworkName | None = None,
     ) -> Run:
         """Submit one rollout, await its terminal trace, and fold it into a ``Run``.
 
@@ -1061,7 +1073,12 @@ async def run(
         trace_id = trace_id or uuid.uuid4().hex
         try:
             state = await self._submit_and_await(
-                task, agent, job_id=job_id, group_id=group_id, trace_id=trace_id
+                task,
+                agent,
+                job_id=job_id,
+                group_id=group_id,
+                trace_id=trace_id,
+                source_framework=source_framework,
             )
         except (TimeoutError, asyncio.CancelledError):
             raise
@@ -1083,6 +1100,7 @@ async def _submit_and_await(
         job_id: str,
         group_id: str | None,
         trace_id: str,
+        source_framework: SourceFrameworkName | None,
     ) -> dict[str, Any]:
         spec_of = getattr(agent, "hosted_spec", None)
         if not callable(spec_of):
@@ -1094,6 +1112,16 @@ async def _submit_and_await(
         platform = PlatformClient.from_settings()
         if not platform.api_key:
             raise RuntimeError("HUD-hosted execution requires HUD_API_KEY")
+        from .source_framework import resolve_source_framework
+
+        source_framework = source_framework or self.source_framework
+        if source_framework is None and self.source is not None:
+            source_framework = resolve_source_framework(self.source)
+        if source_framework is None:
+            raise ValueError(
+                "hosted execution requires source_framework provenance; "
+                "pass source_framework explicitly or provide a source tree with Dockerfile.hud"
+            )
         payload: dict[str, Any] = {
             # The SDK's hex ids travel as canonical UUID strings.
             "trace_id": str(uuid.UUID(trace_id)),
@@ -1102,6 +1130,7 @@ async def _submit_and_await(
             "task": task.id,
             "args": task.args,
             "agent": spec,
+            "source_framework": source_framework,
         }
         if group_id is not None:
             payload["group_id"] = group_id
diff --git a/hud/eval/source_framework.py b/hud/eval/source_framework.py
new file mode 100644
index 000000000..4456ff953
--- /dev/null
+++ b/hud/eval/source_framework.py
@@ -0,0 +1,27 @@
+"""Resolve ``source_framework`` from a local environment tree.
+
+v1: ``Dockerfile.hud`` → ``"hud"``. Callers treat ``None`` as failure.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+SourceFrameworkName = Literal["hud"]
+
+_HUD_MARKER = "Dockerfile.hud"
+
+
+def resolve_source_framework(path: Path | str) -> SourceFrameworkName | None:
+    """Return ``"hud"`` if ``Dockerfile.hud`` exists under *path*, else ``None``."""
+    root = Path(path).expanduser().resolve()
+    if (root / _HUD_MARKER).is_file():
+        return "hud"
+    return None
+
+
+__all__ = [
+    "SourceFrameworkName",
+    "resolve_source_framework",
+]
diff --git a/hud/eval/task.py b/hud/eval/task.py
index 2f6eb2814..92f88083b 100644
--- a/hud/eval/task.py
+++ b/hud/eval/task.py
@@ -33,6 +33,7 @@
 
     from .job import Job
     from .runtime import HostedRuntime, Provider
+    from .source_framework import SourceFrameworkName
 
 
 class Task(BaseModel):
@@ -77,6 +78,7 @@ async def run(
         max_concurrent: int | None = None,
         job: Job | None = None,
         rollout_timeout: float | None = None,
+        source_framework: SourceFrameworkName = "hud",
     ) -> Job:
         """Run this task with ``agent``: the single-task form of ``Taskset.run``.
 
@@ -86,6 +88,8 @@ async def run(
         over a taskset of one. ``runtime`` is the placement; left unset it
         falls back to the HUD runtime tunnel by ``env`` name. For a local
         run, pass one explicitly (``runtime=LocalRuntime("env.py")``).
+        ``source_framework`` declares the environment's provenance. Resolve it
+        from the environment source tree before starting the run.
         """
         from .taskset import Taskset  # circular: taskset -> sync -> task
 
@@ -97,6 +101,7 @@ async def run(
             max_concurrent=max_concurrent,
             job=job,
             rollout_timeout=rollout_timeout,
+            source_framework=source_framework,
         )
 
 
diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py
index 2ef0d223a..c013b49e8 100644
--- a/hud/eval/taskset.py
+++ b/hud/eval/taskset.py
@@ -33,6 +33,7 @@
 
     from .run import Run
     from .runtime import Provider
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 logger = logging.getLogger("hud.eval.taskset")
@@ -248,6 +249,7 @@ async def run(
         max_concurrent: int | None = None,
         job: Job | None = None,
         rollout_timeout: float | None = None,
+        source_framework: SourceFrameworkName = "hud",
     ) -> Job:
         """Run every task x ``group`` with an optional concurrency cap.
 
@@ -273,6 +275,9 @@ async def run(
         as a failed/errored run so one wedged rollout (e.g. a stuck sampling
         stream) cannot stall the whole batch. ``HUDRuntime`` carries its own
         ``run_timeout`` instead.
+
+        ``source_framework`` declares the environments' provenance. Resolve it
+        from the environment source tree before starting the batch.
         """
         group = group or (job.group if job else 1)
         if group < 1:
@@ -312,7 +317,13 @@ async def run(
         async def _run(task: Task, group_id: str) -> Run:
             assert placement is not None  # only reached when tasks were expanded
             if isinstance(placement, HostedRuntime):
-                return await placement.run(task, agent, job_id=job_id, group_id=group_id)
+                return await placement.run(
+                    task,
+                    agent,
+                    job_id=job_id,
+                    group_id=group_id,
+                    source_framework=source_framework,
+                )
             return await rollout(
                 task,
                 agent,
@@ -320,6 +331,7 @@ async def _run(task: Task, group_id: str) -> Run:
                 job_id=job_id,
                 group_id=group_id,
                 rollout_timeout=rollout_timeout,
+                source_framework=source_framework,
             )
 
         async def _one(task: Task, group_id: str) -> Run:
diff --git a/hud/eval/tests/test_chat.py b/hud/eval/tests/test_chat.py
index 47220ccfb..89b3f5ecd 100644
--- a/hud/eval/tests/test_chat.py
+++ b/hud/eval/tests/test_chat.py
@@ -52,12 +52,12 @@ def test_requires_an_agent(self, dummy_task: Any) -> None:
             Chat(dummy_task)  # type: ignore[call-arg]
 
     def test_messages_start_empty_and_are_the_public_history(self, dummy_task: Any) -> None:
-        chat = Chat(dummy_task, _EchoAgent())
+        chat = Chat(dummy_task, _EchoAgent(), source_framework="hud")
         assert chat.messages == []
         assert chat.job is None  # the conversation's job starts on the first send
         # History is the plain ``messages`` list: persist/restore it directly.
         chat.messages = [{"role": "user", "content": {"type": "text", "text": "hi"}}]
-        assert Chat(dummy_task, _EchoAgent()).messages == []
+        assert Chat(dummy_task, _EchoAgent(), source_framework="hud").messages == []
 
 
 _CHAT_ENV = """\
@@ -89,7 +89,12 @@ class TestSend:
     async def test_send_runs_a_turn_and_stores_prompt_message_format(
         self, chat_env_file: Path
     ) -> None:
-        chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file))
+        chat = Chat(
+            _chat_task(),
+            _EchoAgent(),
+            runtime=SubprocessRuntime(chat_env_file),
+            source_framework="hud",
+        )
 
         trace = await chat.send("hello")
 
@@ -107,7 +112,12 @@ async def test_send_runs_a_turn_and_stores_prompt_message_format(
         assert assistant_msg["content"]["text"] == "echo:hello"
 
     async def test_one_job_spans_the_conversation(self, chat_env_file: Path) -> None:
-        chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file))
+        chat = Chat(
+            _chat_task(),
+            _EchoAgent(),
+            runtime=SubprocessRuntime(chat_env_file),
+            source_framework="hud",
+        )
 
         await chat.send("hello")
         await chat.send("again")
@@ -125,7 +135,12 @@ class _Boom(Agent):
             async def __call__(self, run: Any) -> None:
                 raise RuntimeError("agent exploded")
 
-        chat = Chat(_chat_task(), _Boom(), runtime=SubprocessRuntime(chat_env_file))
+        chat = Chat(
+            _chat_task(),
+            _Boom(),
+            runtime=SubprocessRuntime(chat_env_file),
+            source_framework="hud",
+        )
 
         with pytest.raises(RuntimeError, match="agent exploded"):
             await chat.send("hello")
diff --git a/hud/eval/tests/test_hosted.py b/hud/eval/tests/test_hosted.py
index 374aa5481..e14b41627 100644
--- a/hud/eval/tests/test_hosted.py
+++ b/hud/eval/tests/test_hosted.py
@@ -13,7 +13,7 @@
 
 import asyncio
 import uuid
-from typing import Any, ClassVar, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 import pytest
 
@@ -34,6 +34,16 @@
 from hud.settings import settings
 from hud.telemetry.context import set_trace_context
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@pytest.fixture
+def hud_env(tmp_path: Path) -> Path:
+    """Minimal env tree that resolves ``source_framework`` to ``hud``."""
+    (tmp_path / "Dockerfile.hud").write_text("FROM python:3.11\n")
+    return tmp_path
+
 
 class _FakePlatform:
     """Scripted PlatformClient: records posts, serves trace states in order."""
@@ -112,7 +122,9 @@ async def test_run_rejects_non_gateway_agent() -> None:
 
 
 @pytest.mark.asyncio
-async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_run_submits_and_polls_to_terminal(
+    monkeypatch: pytest.MonkeyPatch, hud_env: Path
+) -> None:
     platform = _FakePlatform(
         [
             {"status": "pending"},
@@ -124,7 +136,7 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch
         "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
     )
 
-    hosted = HostedRuntime(poll_interval=0.0)
+    hosted = HostedRuntime(source=hud_env, poll_interval=0.0)
     trace_id = uuid.uuid4().hex
     job_id = uuid.uuid4().hex
     task = Task(
@@ -160,6 +172,7 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch
         "limits": {"startup_timeout_s": 120, "run_timeout_s": 900},
     }
     assert payload["group_id"] == "g1"
+    assert payload["source_framework"] == "hud"
     assert payload["agent"]["type"] == "openai_compatible"
     assert payload["agent"]["config"]["model"] == "test-model"
 
@@ -167,13 +180,14 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch
 @pytest.mark.asyncio
 async def test_run_preserves_runtime_config_null_override(
     monkeypatch: pytest.MonkeyPatch,
+    hud_env: Path,
 ) -> None:
     platform = _FakePlatform([{"status": "completed", "reward": 0.5}])
     monkeypatch.setattr(
         "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
     )
 
-    await HostedRuntime(poll_interval=0.0).run(
+    await HostedRuntime(source=hud_env, poll_interval=0.0).run(
         Task(env="sums", id="add", runtime_config=RuntimeConfig(resources=None)),
         _agent(),
         job_id=uuid.uuid4().hex,
@@ -184,13 +198,15 @@ async def test_run_preserves_runtime_config_null_override(
 
 
 @pytest.mark.asyncio
-async def test_run_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_run_timeout_requests_platform_cancel(
+    monkeypatch: pytest.MonkeyPatch, hud_env: Path
+) -> None:
     platform = _FakePlatform([{"status": "running"}])
     monkeypatch.setattr(
         "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
     )
 
-    hosted = HostedRuntime(poll_interval=0.0, run_timeout=0.0)
+    hosted = HostedRuntime(source=hud_env, poll_interval=0.0, run_timeout=0.0)
     task = Task(env="sums", id="add", args={})
 
     with pytest.raises(TimeoutError, match="hosted rollout"):
@@ -201,14 +217,16 @@ async def test_run_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPa
 
 
 @pytest.mark.asyncio
-async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch, hud_env: Path) -> None:
     platform = _FakePlatform([{"status": "completed", "reward": 1.0, "error": None}])
     monkeypatch.setattr(
         "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
     )
 
     task = Task(env="sums", id="add", args={"a": 2, "b": 3})
-    run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex)
+    run = await HostedRuntime(source=hud_env, poll_interval=0.0).run(
+        task, _agent(), job_id=uuid.uuid4().hex
+    )
 
     assert run.reward == 1.0
     assert run.trace.status == "completed"
@@ -220,20 +238,57 @@ async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch) -> N
 
 
 @pytest.mark.asyncio
-async def test_run_folds_error_receipt(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_run_folds_error_receipt(monkeypatch: pytest.MonkeyPatch, hud_env: Path) -> None:
     platform = _FakePlatform([{"status": "error", "reward": None, "error": "env exploded"}])
     monkeypatch.setattr(
         "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
     )
 
     task = Task(env="sums", id="add", args={})
-    run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex)
+    run = await HostedRuntime(source=hud_env, poll_interval=0.0).run(
+        task, _agent(), job_id=uuid.uuid4().hex
+    )
 
     assert run.reward == 0.0
     assert run.trace.is_error
     assert "env exploded" in (run.trace.error or "")
 
 
+@pytest.mark.asyncio
+async def test_run_fails_when_source_framework_unresolved(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    platform = _FakePlatform([{"status": "completed", "reward": 1.0}])
+    monkeypatch.setattr(
+        "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
+    )
+
+    run = await HostedRuntime(source=tmp_path, poll_interval=0.0).run(
+        Task(env="sums", id="add"), _agent(), job_id=uuid.uuid4().hex
+    )
+
+    assert run.trace.is_error
+    assert "source_framework provenance" in (run.trace.error or "")
+    assert platform.posts == []
+
+
+@pytest.mark.asyncio
+async def test_run_accepts_explicit_framework_without_local_source(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    platform = _FakePlatform([{"status": "completed", "reward": 1.0}])
+    monkeypatch.setattr(
+        "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform)
+    )
+
+    run = await HostedRuntime(source_framework="hud", poll_interval=0.0).run(
+        Task(env="sums", id="add"), _agent(), job_id=uuid.uuid4().hex
+    )
+
+    assert run.trace.status == "completed"
+    assert platform.posts[0][1]["source_framework"] == "hud"
+
+
 @pytest.mark.asyncio
 async def test_scheduler_drives_provider_locally(monkeypatch: pytest.MonkeyPatch) -> None:
     """A Provider placement goes through the local rollout atom, not HostedRuntime."""
@@ -251,7 +306,7 @@ async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run:
     monkeypatch.setattr(taskset_mod, "rollout", fake_rollout)
 
     job = await Taskset("t", [Task(env="e", id="x")]).run(
-        _agent(), runtime=Runtime("tcp://127.0.0.1:1")
+        _agent(), runtime=Runtime("tcp://127.0.0.1:1"), source_framework="hud"
     )
 
     assert len(job.runs) == 1
@@ -274,11 +329,12 @@ async def run(self, task: Task, agent: Any, **kwargs: Any) -> Run:  # type: igno
             return run
 
     job = await Taskset("t", [Task(env="e", id="x")]).run(
-        _agent(), runtime=_RecordingHostedRuntime()
+        _agent(), runtime=_RecordingHostedRuntime(), source_framework="hud"
     )
 
     assert len(job.runs) == 1
     assert "job_id" in seen and "group_id" in seen
+    assert seen["source_framework"] == "hud"
 
 
 @pytest.mark.asyncio
diff --git a/hud/eval/tests/test_job.py b/hud/eval/tests/test_job.py
index 9b61df8b5..ad8bf8781 100644
--- a/hud/eval/tests/test_job.py
+++ b/hud/eval/tests/test_job.py
@@ -46,6 +46,16 @@ def _run_with(trace_id: str, *, extra: dict[str, Any]) -> Run:
     return run
 
 
+async def test_trace_enter_defaults_source_framework_to_hud(recorder: _Recorder) -> None:
+    await job_mod.trace_enter("abc", job_id="j", group_id=None, model="m")
+
+    assert len(recorder.calls) == 1
+    path, body = recorder.calls[0]
+    assert path == "/trace/abc/enter"
+    assert body["source_framework"] == "hud"
+    assert body["model"] == "m"
+
+
 async def test_trace_exit_propagates_stop_reason(recorder: _Recorder) -> None:
     run = _run_with("abc", extra={})
     run.trace.stop_reason = "max_steps"
diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py
index fd8ec4240..3028e0fd0 100644
--- a/hud/eval/tests/test_local_runtime.py
+++ b/hud/eval/tests/test_local_runtime.py
@@ -106,6 +106,7 @@ def _solve(prompt: str) -> str:
         _FnAgent(_solve),
         runtime=LocalRuntime(tmp_path / "env.py"),
         group=2,
+        source_framework="hud",
     )
 
     assert [run.reward for run in job.runs] == [1.0, 1.0]
@@ -116,6 +117,7 @@ async def test_live_env_pointer_resolves_to_its_declaring_file(imported_env) ->
         Task(env="sums", id="add", args={"a": 2, "b": 3}),
         _FnAgent(_solve_add),
         runtime=LocalRuntime(imported_env.env),
+        source_framework="hud",
     )
 
     assert run.reward == 1.0
@@ -138,6 +140,7 @@ def env_for(task: Task) -> Environment:
         runtime=LocalRuntime(env_for),
         group=3,
         max_concurrent=3,
+        source_framework="hud",
     )
 
     assert all(run.reward == 1.0 for run in job.runs)
@@ -170,7 +173,7 @@ async def test_module_loaded_taskset_serves_its_source_by_default(tmp_path, requ
     request.addfinalizer(lambda: sys.modules.pop("env", None))
 
     taskset = Taskset.from_module(tmp_path / "tasks.py")
-    job = await taskset.run(_FnAgent(_solve_add))
+    job = await taskset.run(_FnAgent(_solve_add), source_framework="hud")
 
     assert len(job.runs) == 2
     assert all(run.reward == 1.0 for run in job.runs)
@@ -178,7 +181,7 @@ async def test_module_loaded_taskset_serves_its_source_by_default(tmp_path, requ
 
 async def test_minted_tasks_resolve_a_declared_env_by_name(imported_env) -> None:
     job = await Taskset("sums", [imported_env.add(a=2, b=3), imported_env.add(a=4, b=5)]).run(
-        _FnAgent(_solve_add)
+        _FnAgent(_solve_add), source_framework="hud"
     )
 
     assert len(job.runs) == 2
@@ -187,7 +190,7 @@ async def test_minted_tasks_resolve_a_declared_env_by_name(imported_env) -> None
 
 async def test_no_placement_fails_with_the_forms_to_pass() -> None:
     with pytest.raises(ValueError, match="no placement for env"):
-        await Task(env="ghost", id="add").run(_FnAgent(_solve_add))
+        await Task(env="ghost", id="add").run(_FnAgent(_solve_add), source_framework="hud")
 
 
 async def test_ambiguous_env_name_fails_loudly(imported_env, tmp_path, request) -> None:
@@ -201,7 +204,7 @@ async def test_ambiguous_env_name_fails_loudly(imported_env, tmp_path, request)
     try:
         spec.loader.exec_module(module)
         with pytest.raises(ValueError, match="LocalRuntime\\(env\\)"):
-            await Task(env="sums", id="add").run(_FnAgent(_solve_add))
+            await Task(env="sums", id="add").run(_FnAgent(_solve_add), source_framework="hud")
     finally:
         del sys.modules[module_name]
 
@@ -255,7 +258,7 @@ async def test_tasks_only_module_resolves_envs_imported_from_elsewhere(
     request.addfinalizer(lambda: sys.modules.pop("sums_envmod", None))
 
     taskset = Taskset.from_module(tasks_dir / "tasks.py")
-    job = await taskset.run(_FnAgent(_solve_add))
+    job = await taskset.run(_FnAgent(_solve_add), source_framework="hud")
 
     assert [run.reward for run in job.runs] == [1.0]
 
@@ -267,13 +270,13 @@ async def test_single_file_taskset_never_drags_in_a_same_named_sibling(tmp_path)
     (tmp_path / "env_b.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
 
     taskset = Taskset.from_module(tmp_path / "env_a.py")
-    job = await taskset.run(_FnAgent(_solve_add))
+    job = await taskset.run(_FnAgent(_solve_add), source_framework="hud")
 
     assert [run.reward for run in job.runs] == [1.0]
 
 
 async def test_empty_taskset_runs_without_a_placement() -> None:
-    job = await Taskset("empty", []).run(_FnAgent(_solve_add))
+    job = await Taskset("empty", []).run(_FnAgent(_solve_add), source_framework="hud")
 
     assert job.runs == []
 
@@ -311,7 +314,7 @@ def _solve(prompt: str) -> str:
         return str(int(a) + int(b))
 
     taskset = Taskset.from_module(tasks_dir / "tasks.py")
-    job = await taskset.run(_FnAgent(_solve), group=2)
+    job = await taskset.run(_FnAgent(_solve), group=2, source_framework="hud")
 
     assert [run.reward for run in job.runs] == [1.0, 1.0]
 
@@ -335,6 +338,7 @@ async def test_package_reexported_env_pointer_uses_the_declaring_submodule(
         Task(env="sums", id="add", args={"a": 2, "b": 3}),
         _FnAgent(_solve_add),
         runtime=LocalRuntime(sums_pkg.env),
+        source_framework="hud",
     )
 
     assert run.reward == 1.0
@@ -362,6 +366,7 @@ def _solve(prompt: str) -> str:
         runtime=LocalRuntime(tmp_path / "env.py"),
         group=2,
         max_concurrent=2,
+        source_framework="hud",
     )
 
     assert [run.reward for run in job.runs] == [1.0, 1.0]
diff --git a/hud/eval/tests/test_rollout.py b/hud/eval/tests/test_rollout.py
index 3473d5a1c..f524b269d 100644
--- a/hud/eval/tests/test_rollout.py
+++ b/hud/eval/tests/test_rollout.py
@@ -155,7 +155,12 @@ async def _wait_for_pid_inactive(pid: int, max_wait: float = 2.0) -> bool:
 
 
 async def test_rollout_returns_graded_run_with_trace_id(env_file: Path) -> None:
-    run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file))
+    run = await rollout(
+        _add_task(2, 3),
+        _FnAgent(_solve_add),
+        runtime=SubprocessRuntime(env_file),
+        source_framework="hud",
+    )
 
     assert run.reward == 1.0
     assert run.trace.content == "5"
@@ -200,6 +205,7 @@ async def write_report():
         Task(env="opencode_report", id="write_report"),
         agent,
         runtime=lambda _task: _local(env),
+        source_framework="hud",
     )
 
     assert run.reward == 1.0
@@ -263,7 +269,12 @@ async def test_mid_run_failure_keeps_the_real_run_and_its_evidence(env_file: Pat
     def boom(prompt: str) -> str:
         raise RuntimeError("agent exploded")
 
-    run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=SubprocessRuntime(env_file))
+    run = await rollout(
+        _add_task(2, 3),
+        _FnAgent(boom),
+        runtime=SubprocessRuntime(env_file),
+        source_framework="hud",
+    )
 
     assert run.trace.is_error
     assert "agent exploded" in (run.trace.error or "")
@@ -291,7 +302,10 @@ async def test_mid_run_failure_still_grades_best_effort(env_file: Path) -> None:
     # The agent answers correctly, then fails. The env is still alive, so the
     # run is graded best-effort: the reward is captured even though it errored.
     run = await rollout(
-        _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=SubprocessRuntime(env_file)
+        _add_task(2, 3),
+        _AnswerThenBoomAgent(_solve_add),
+        runtime=SubprocessRuntime(env_file),
+        source_framework="hud",
     )
 
     assert run.trace.is_error
@@ -326,6 +340,7 @@ async def add(a: int, b: int):
         _SlowAgent(_solve_add),
         runtime=lambda _row: _local(env),
         rollout_timeout=0.5,
+        source_framework="hud",
     )
 
     # The deadline fired mid-loop, but the run was live with an answer already
@@ -342,7 +357,12 @@ async def broken_provider(task: TaskRow) -> AsyncIterator[Runtime]:
         raise RuntimeError("no substrate for you")
         yield  # pragma: no cover
 
-    run = await rollout(_add_task(1, 1), _FnAgent(_solve_add), runtime=broken_provider)
+    run = await rollout(
+        _add_task(1, 1),
+        _FnAgent(_solve_add),
+        runtime=broken_provider,
+        source_framework="hud",
+    )
 
     assert run.trace.is_error
     assert "no substrate for you" in (run.trace.error or "")
@@ -360,14 +380,20 @@ def placer(task: TaskRow) -> Any:
         placed.append(f"{task.env}/{task.id}:{task.args['a']}")
         return SubprocessRuntime(env_file)(task)
 
-    run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=placer)
+    run = await rollout(
+        _add_task(2, 3), _FnAgent(_solve_add), runtime=placer, source_framework="hud"
+    )
 
     assert run.reward == 1.0
     assert placed == ["sums/add:2"]
 
 
 async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None:
-    job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(env_file))
+    job = await _add_task(2, 3).run(
+        _FnAgent(_solve_add),
+        runtime=SubprocessRuntime(env_file),
+        source_framework="hud",
+    )
 
     (run,) = job.runs
     assert job.reward == 1.0
@@ -377,7 +403,11 @@ async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None:
 
 async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None:
     job = await _add_task(1, 2).run(
-        _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file), group=2, max_concurrent=1
+        _FnAgent(_solve_add),
+        runtime=SubprocessRuntime(env_file),
+        group=2,
+        max_concurrent=1,
+        source_framework="hud",
     )
 
     assert job.group == 2
@@ -390,8 +420,12 @@ async def test_open_job_spans_multiple_scheduler_calls(env_file: Path) -> None:
     session = await Job.start("session", group=2)
     provider = SubprocessRuntime(env_file)
 
-    job1 = await _add_task(1, 1).run(_FnAgent(_solve_add), runtime=provider, job=session)
-    job2 = await _add_task(2, 2).run(_FnAgent(_solve_add), runtime=provider, job=session)
+    job1 = await _add_task(1, 1).run(
+        _FnAgent(_solve_add), runtime=provider, job=session, source_framework="hud"
+    )
+    job2 = await _add_task(2, 2).run(
+        _FnAgent(_solve_add), runtime=provider, job=session, source_framework="hud"
+    )
 
     # Both calls accumulate into the one open job (group defaults to the job's).
     assert job1 is session
@@ -434,7 +468,9 @@ async def test_one_spawn_serves_each_rows_env_in_a_mixed_taskset(
     # One provider, two envs: each acquisition serves the row it was called
     # with (the task ids only exist on their own env, so a misplacement
     # would fail the rollout).
-    job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(path))
+    job = await Taskset("zoo", rows).run(
+        _FnAgent(_solve_add), runtime=SubprocessRuntime(path), source_framework="hud"
+    )
 
     assert [run.reward for run in job.runs] == [1.0, 1.0]
     assert [run.prompt for run in job.runs] == ["alpha:1:2", "beta:3:4"]
@@ -447,6 +483,7 @@ async def test_rollout_threads_job_and_group_ids(env_file: Path) -> None:
         runtime=SubprocessRuntime(env_file),
         job_id="j1",
         group_id="g1",
+        source_framework="hud",
     )
 
     assert run.reward == 1.0
diff --git a/hud/eval/tests/test_source_framework.py b/hud/eval/tests/test_source_framework.py
new file mode 100644
index 000000000..a994b5ad8
--- /dev/null
+++ b/hud/eval/tests/test_source_framework.py
@@ -0,0 +1,25 @@
+"""``resolve_source_framework``: Dockerfile.hud marker detection."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from hud.eval.source_framework import resolve_source_framework
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def test_resolve_returns_hud_when_dockerfile_hud_present(tmp_path: Path) -> None:
+    (tmp_path / "Dockerfile.hud").write_text("FROM python:3.11\n")
+    assert resolve_source_framework(tmp_path) == "hud"
+
+
+def test_resolve_returns_none_without_marker(tmp_path: Path) -> None:
+    (tmp_path / "Dockerfile").write_text("FROM python:3.11\n")
+    (tmp_path / "pyproject.toml").write_text("[project]\nname='x'\n")
+    assert resolve_source_framework(tmp_path) is None
+
+
+def test_resolve_returns_none_for_empty_dir(tmp_path: Path) -> None:
+    assert resolve_source_framework(tmp_path) is None
diff --git a/hud/eval/tests/test_task.py b/hud/eval/tests/test_task.py
index 5d0de0586..133c28333 100644
--- a/hud/eval/tests/test_task.py
+++ b/hud/eval/tests/test_task.py
@@ -152,7 +152,7 @@ async def fake_rollout(task: Task, agent: Agent, **kwargs: object) -> Run:
 
     task = Task(env="hosted-env", id="solve", args={"n": 1})
     taskset = taskset_mod.Taskset("hosted", [task], origin="api:ts_123")
-    job = await taskset.run(cast("Agent", object()))
+    job = await taskset.run(cast("Agent", object()), source_framework="hud")
 
     (run,) = job.runs
     assert run.trace.status == "completed"

From dd89244aec1826714a7de3c69ac322d7aa77e100 Mon Sep 17 00:00:00 2001
From: howenyap <wenhyap02@gmail.com>
Date: Fri, 17 Jul 2026 11:31:00 +0800
Subject: [PATCH 2/2] fix(eval): fall back to hud source framework without
 local marker

Local task files without a Dockerfile.hud marker (e.g. envs served via
--runtime hud or tcp://) now use the SDK default instead of aborting.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 hud/cli/eval.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hud/cli/eval.py b/hud/cli/eval.py
index b17a38dff..2593d8567 100644
--- a/hud/cli/eval.py
+++ b/hud/cli/eval.py
@@ -843,9 +843,10 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
         from hud.eval import resolve_source_framework
 
         source_root = source_path if source_path.is_dir() else source_path.parent
-        source_framework = resolve_source_framework(source_root)
-        if source_framework is None:
-            raise ValueError(f"could not determine source framework from {source_root}")
+        # No marker file just means the local path only holds tasks (e.g. the
+        # env runs behind --runtime hud or a tcp:// url); use the SDK default
+        # rather than refusing to run.
+        source_framework = resolve_source_framework(source_root) or "hud"
     else:
         # Platform tasksets currently contain HUD-authored environments.
         source_framework = "hud"