feat(eval): propagate trace source framework

hud/cli/eval.py

@@ -740,7 +740,13 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
 
     if cfg.remote:
         require_api_key("run remote hosted evals")
-        return HostedRuntime()
+        # A platform taskset has no local source tree. All currently exported
+        # platform tasksets are HUD-authored; future export records can carry
+        # their framework explicitly here.
+        return HostedRuntime(
+            source=source_path,
+            source_framework="hud" if source_path is None else None,
+        )
     if cfg.runtime == "local":
         if source_path is None:
             raise ValueError("local placement requires a local source path")
@@ -833,12 +839,24 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
 
     agent = _build_agent(cfg)
     placement = _resolve_placement(cfg, source_path if is_local else None)
+    if is_local:
+        from hud.eval import resolve_source_framework
+
+        source_root = source_path if source_path.is_dir() else source_path.parent
+        # No marker file just means the local path only holds tasks (e.g. the
+        # env runs behind --runtime hud or a tcp:// url); use the SDK default
+        # rather than refusing to run.
+        source_framework = resolve_source_framework(source_root) or "hud"
+    else:
+        # Platform tasksets currently contain HUD-authored environments.
+        source_framework = "hud"
 
     job = await taskset.run(
         agent,
         runtime=placement,
         group=cfg.group_size,
         max_concurrent=cfg.max_concurrent,
+        source_framework=source_framework,
     )
     if job.runs and settings.telemetry_enabled and settings.api_key:
         hud_console.info(f"{settings.hud_web_url}/jobs/{job.id}")

hud/cli/tests/test_eval_config.py

@@ -148,6 +148,23 @@ def test_resolve_placement_remote_uses_hosted_runtime(
     placement = eval_mod._resolve_placement(EvalConfig(remote=True), tmp_path)
 
     assert isinstance(placement, HostedRuntime)
+    assert placement.source == tmp_path
+    assert placement.source_framework is None
+
+
+def test_resolve_placement_remote_platform_taskset_declares_hud_framework(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from hud.eval import HostedRuntime
+    from hud.settings import settings
+
+    monkeypatch.setattr(settings, "api_key", "sk-hud-test")
+
+    placement = eval_mod._resolve_placement(EvalConfig(remote=True), None)
+
+    assert isinstance(placement, HostedRuntime)
+    assert placement.source is None
+    assert placement.source_framework == "hud"
 
 
 def test_runtime_cli_override_clears_config_remote() -> None:

hud/environment/tests/test_legacy.py

@@ -120,7 +120,11 @@ async def test_taskset_concurrent_grouped_rollouts() -> None:
     taskset = Taskset("adds", (add(a=i, b=i + 1) for i in range(4)))
 
     job = await taskset.run(
-        _FnAgent(_solve_add), runtime=lambda _row: _local(env), group=2, max_concurrent=3
+        _FnAgent(_solve_add),
+        runtime=lambda _row: _local(env),
+        group=2,
+        max_concurrent=3,
+        source_framework="hud",
     )
     runs = job.runs
 
@@ -144,7 +148,7 @@ def solve_or_boom(prompt: str) -> str:
         return _solve_add(prompt)
 
     job = await Taskset("adds", (add(a=i, b=1) for i in range(4))).run(
-        _FnAgent(solve_or_boom), runtime=lambda _row: _local(env)
+        _FnAgent(solve_or_boom), runtime=lambda _row: _local(env), source_framework="hud"
     )
     runs = job.runs
 

hud/eval/__init__.py

@@ -48,6 +48,7 @@
     RuntimeResources,
     SubprocessRuntime,
 )
+from .source_framework import resolve_source_framework
 from .sync import SyncPlan
 from .task import Task
 from .taskset import Taskset
@@ -74,5 +75,6 @@
     "Task",
     "Taskset",
     "Trace",
+    "resolve_source_framework",
     "rollout",
 ]

hud/eval/chat.py

@@ -11,7 +11,11 @@
     from hud.agents import create_agent
     from tasks import assistant  # an @env.template taking ``messages``
 
-    chat = Chat(assistant(messages=[]), create_agent("claude-sonnet-4-5"))
+    chat = Chat(
+        assistant(messages=[]),
+        create_agent("claude-sonnet-4-5"),
+        source_framework="hud",
+    )
     r1 = await chat.send("Book me a flight")
     r2 = await chat.send("SFO to JFK")
 
@@ -38,6 +42,7 @@
     from hud.agents.base import Agent
 
     from .runtime import Provider
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 LOGGER = logging.getLogger(__name__)
@@ -84,6 +89,7 @@ def __init__(
         /,
         *,
         runtime: Provider | None = None,
+        source_framework: SourceFrameworkName = "hud",
     ) -> None:
         """Initialize Chat.
 
@@ -99,10 +105,13 @@ def __init__(
                 ``LocalRuntime("env.py")`` or ``Runtime("tcp://...")``. Chat is
                 interactive and local: it drives the agent loop in this process,
                 so hosted placement does not apply.
+            source_framework: Explicit provenance for a served runtime whose
+                local source tree is unavailable.
         """
         self._task = task
         self._agent = agent
         self._runtime = runtime
+        self._source_framework: SourceFrameworkName = source_framework
         self.messages: list[dict[str, Any]] = []
         #: The conversation's job — every turn's run reports under it
         #: (started on the first ``send``).
@@ -138,7 +147,13 @@ async def send(self, message: MessageContent) -> Trace:
             )
         if self.job is None:  # one job spans the whole conversation
             self.job = await Job.start(self._task.id)
-        run = await rollout(task, self._agent, runtime=self._runtime, job_id=self.job.id)
+        run = await rollout(
+            task,
+            self._agent,
+            runtime=self._runtime,
+            job_id=self.job.id,
+            source_framework=self._source_framework,
+        )
         self.job.runs.append(run)
         result = run.trace
         if result.is_error:

hud/eval/job.py

@@ -105,18 +105,26 @@ async def trace_enter(
     *,
     job_id: str | None,
     group_id: str | None,
+    source_framework: str = "hud",
     model: str | None = None,
 ) -> None:
     """Report that one rollout started.
 
     ``model`` is the model string the agent will sample (when known); the
     platform resolves it and attributes the trace immediately on enter.
+    Resolve ``source_framework`` from the environment source tree before
+    entering the trace.
     """
     if not _reporting_enabled():
         return
     await _report(
         f"/trace/{trace_id}/enter",
-        {"job_id": job_id, "group_id": group_id, "model": model},
+        {
+            "job_id": job_id,
+            "group_id": group_id,
+            "model": model,
+            "source_framework": source_framework,
+        },
     )
 
 

hud/eval/run.py

@@ -5,7 +5,12 @@
 loopback, a container, a cloud sandbox), starts the task, drives the agent,
 grades, and tears down, filling a :class:`Run` along the way::
 
-    run = await rollout(task, agent, runtime=LocalRuntime("env.py"))
+    run = await rollout(
+        task,
+        agent,
+        runtime=LocalRuntime("env.py"),
+        source_framework="hud",
+    )
 
 It is the *client-here* path: the agent loop runs in this process against a
 :class:`~hud.eval.runtime.Provider`'s channel. The same driver runs on the
@@ -45,6 +50,7 @@
     from hud.clients.client import HudClient
 
     from .runtime import Provider, Runtime
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 logger = logging.getLogger("hud.eval.run")
@@ -283,6 +289,7 @@ async def rollout(
     group_id: str | None = None,
     trace_id: str | None = None,
     rollout_timeout: float | None = None,
+    source_framework: SourceFrameworkName = "hud",
 ) -> Run:
     """Drive one task to a graded :class:`Run` here, against ``runtime``'s channel.
 
@@ -321,7 +328,13 @@ async def rollout(
 
     agent_model = agent.config.model if isinstance(agent, ToolAgent) else None
     with set_trace_context(trace_id):
-        await trace_enter(trace_id, job_id=job_id, group_id=group_id, model=agent_model)
+        await trace_enter(
+            trace_id,
+            job_id=job_id,
+            group_id=group_id,
+            model=agent_model,
+            source_framework=source_framework,
+        )
         run: Run | None = None
         _phase = "provisioning"
 

hud/eval/runtime.py

@@ -58,6 +58,7 @@
     from hud.agents.base import Agent
     from hud.environment.env import Environment
 
+    from .source_framework import SourceFrameworkName
     from .task import Task
 
 logger = logging.getLogger("hud.eval.runtime")
@@ -898,6 +899,7 @@ class HUDRuntime:
     def __init__(self, *, run_timeout: float = 3600.0, runtime_url: str | None = None) -> None:
         self.run_timeout = run_timeout
         self.runtime_url = runtime_url
+        self.source_framework: SourceFrameworkName = "hud"
 
     async def run(
         self,
@@ -915,6 +917,7 @@ async def run(
             trace_id=trace_id,
             job_id=job_id,
             group_id=group_id,
+            source_framework=self.source_framework,
         )
 
     def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]:
@@ -1034,9 +1037,17 @@ class HostedRuntime:
     def __init__(
         self,
         *,
+        source: str | Path | None = None,
+        source_framework: SourceFrameworkName | None = None,
         poll_interval: float = 5.0,
         run_timeout: float = 3600.0,
     ) -> None:
+        # Directory used to resolve ``source_framework`` (expects Dockerfile.hud).
+        self.source: Path | None = None
+        if source is not None:
+            path = Path(source).expanduser().resolve()
+            self.source = path if path.is_dir() else path.parent
+        self.source_framework: SourceFrameworkName | None = source_framework
         self.poll_interval = poll_interval
         self.run_timeout = run_timeout
 
@@ -1048,6 +1059,7 @@ async def run(
         job_id: str,
         group_id: str | None = None,
         trace_id: str | None = None,
+        source_framework: SourceFrameworkName | None = None,
     ) -> Run:
         """Submit one rollout, await its terminal trace, and fold it into a ``Run``.
 
@@ -1061,7 +1073,12 @@ async def run(
         trace_id = trace_id or uuid.uuid4().hex
         try:
             state = await self._submit_and_await(
-                task, agent, job_id=job_id, group_id=group_id, trace_id=trace_id
+                task,
+                agent,
+                job_id=job_id,
+                group_id=group_id,
+                trace_id=trace_id,
+                source_framework=source_framework,
             )
         except (TimeoutError, asyncio.CancelledError):
             raise
@@ -1083,6 +1100,7 @@ async def _submit_and_await(
         job_id: str,
         group_id: str | None,
         trace_id: str,
+        source_framework: SourceFrameworkName | None,
     ) -> dict[str, Any]:
         spec_of = getattr(agent, "hosted_spec", None)
         if not callable(spec_of):
@@ -1094,6 +1112,16 @@ async def _submit_and_await(
         platform = PlatformClient.from_settings()
         if not platform.api_key:
             raise RuntimeError("HUD-hosted execution requires HUD_API_KEY")
+        from .source_framework import resolve_source_framework
+
+        source_framework = source_framework or self.source_framework
+        if source_framework is None and self.source is not None:
+            source_framework = resolve_source_framework(self.source)
+        if source_framework is None:
+            raise ValueError(
+                "hosted execution requires source_framework provenance; "
+                "pass source_framework explicitly or provide a source tree with Dockerfile.hud"
+            )
         payload: dict[str, Any] = {
             # The SDK's hex ids travel as canonical UUID strings.
             "trace_id": str(uuid.UUID(trace_id)),
@@ -1102,6 +1130,7 @@ async def _submit_and_await(
             "task": task.id,
             "args": task.args,
             "agent": spec,
+            "source_framework": source_framework,
         }
         if group_id is not None:
             payload["group_id"] = group_id

hud/eval/source_framework.py

@@ -0,0 +1,27 @@
+"""Resolve ``source_framework`` from a local environment tree.
+
+v1: ``Dockerfile.hud`` → ``"hud"``. Callers treat ``None`` as failure.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+SourceFrameworkName = Literal["hud"]
+
+_HUD_MARKER = "Dockerfile.hud"
+
+
+def resolve_source_framework(path: Path | str) -> SourceFrameworkName | None:
+    """Return ``"hud"`` if ``Dockerfile.hud`` exists under *path*, else ``None``."""
+    root = Path(path).expanduser().resolve()
+    if (root / _HUD_MARKER).is_file():
+        return "hud"
+    return None
+
+
+__all__ = [
+    "SourceFrameworkName",
+    "resolve_source_framework",
+]

hud/eval/task.py

@@ -33,6 +33,7 @@
 
     from .job import Job
     from .runtime import HostedRuntime, Provider
+    from .source_framework import SourceFrameworkName
 
 
 class Task(BaseModel):
@@ -77,6 +78,7 @@ async def run(
         max_concurrent: int | None = None,
         job: Job | None = None,
         rollout_timeout: float | None = None,
+        source_framework: SourceFrameworkName = "hud",
     ) -> Job:
         """Run this task with ``agent``: the single-task form of ``Taskset.run``.
 
@@ -86,6 +88,8 @@ async def run(
         over a taskset of one. ``runtime`` is the placement; left unset it
         falls back to the HUD runtime tunnel by ``env`` name. For a local
         run, pass one explicitly (``runtime=LocalRuntime("env.py")``).
+        ``source_framework`` declares the environment's provenance. Resolve it
+        from the environment source tree before starting the run.
         """
         from .taskset import Taskset  # circular: taskset -> sync -> task
 
@@ -97,6 +101,7 @@ async def run(
             max_concurrent=max_concurrent,
             job=job,
             rollout_timeout=rollout_timeout,
+            source_framework=source_framework,
         )