From 34b4ac89664ad78014f95117aa5aa582714c3b18 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:19:08 -0700 Subject: [PATCH 1/9] feat(eval): LocalRuntime serves live envs in-process; SubprocessRuntime serves sources Live Environment objects become a placement, not data: the new LocalRuntime accepts an env or a mapping of env name -> env/factory and serves it in this process over the same control channel as every placement. Rows join by env name; an instance is a shared substrate (daemons refcounted across acquisitions, one bound channel per rollout so concurrent task lifecycles never collide), a factory is fresh per acquisition. The child-process provider keeps its exact semantics under its honest name, SubprocessRuntime. Task rows are now pure data all the way: the _source private attr is gone, template factories no longer capture their defining file, and Task/Taskset run defaults collapse to runtime= else the HUD tunnel by env name. _local splits into _bind_channel (channel over a started env) + the single-use start/serve/stop form. --- docs/v6/guides/running-an-eval.mdx | 7 +- docs/v6/guides/training-agents.mdx | 6 +- docs/v6/reference/agents.mdx | 4 +- docs/v6/reference/environment.mdx | 2 +- docs/v6/reference/runtime.mdx | 42 +++++-- docs/v6/reference/tasks.mdx | 4 +- docs/v6/reference/types.mdx | 2 +- docs/v6/start/overview.mdx | 9 +- hud/__init__.py | 2 + hud/cli/eval.py | 6 +- hud/cli/task.py | 4 +- hud/cli/templates.py | 6 +- hud/environment/__init__.py | 3 +- hud/environment/env.py | 10 +- hud/environment/server.py | 2 +- hud/eval/__init__.py | 17 +-- hud/eval/chat.py | 5 +- hud/eval/run.py | 2 +- hud/eval/runtime.py | 145 ++++++++++++++++++--- hud/eval/task.py | 16 +-- hud/eval/taskset.py | 14 +-- hud/eval/tests/test_chat.py | 10 +- hud/eval/tests/test_local_runtime.py | 180 +++++++++++++++++++++++++++ hud/eval/tests/test_rollout.py | 24 ++-- hud/tests/test_init.py | 1 + hud/tests/test_init_module.py | 1 + 26 files changed, 416 insertions(+), 108 deletions(-) create mode 100644 hud/eval/tests/test_local_runtime.py diff --git a/docs/v6/guides/running-an-eval.mdx b/docs/v6/guides/running-an-eval.mdx index 15c309f3c..b83131bfd 100644 --- a/docs/v6/guides/running-an-eval.mdx +++ b/docs/v6/guides/running-an-eval.mdx @@ -112,14 +112,14 @@ is the same eval `hud eval` runs, written out in Python. ```python run.py import asyncio -from hud import Taskset, LocalRuntime +from hud import Taskset, SubprocessRuntime from hud.agents import create_agent agent = create_agent("claude-sonnet-4-5") ts = Taskset.from_file("tasks.py") async def main(): - job = await ts.run(agent, runtime=LocalRuntime("env.py")) + job = await ts.run(agent, runtime=SubprocessRuntime("env.py")) print(job.reward) asyncio.run(main()) @@ -130,7 +130,8 @@ change to `env.py` or the tasks: | Runtime | Where the env runs | | --- | --- | -| `LocalRuntime("env.py")` | A child process on your machine | +| `LocalRuntime(env)` | This process, serving a live env object | +| `SubprocessRuntime("env.py")` | A child process on your machine | | `DockerRuntime("my-env")` | A fresh local container per rollout | | `ModalRuntime("my-env")` | A fresh [Modal](https://modal.com) sandbox per rollout | | `DaytonaRuntime("my-env")` | A fresh [Daytona](https://daytona.io) sandbox per rollout | diff --git a/docs/v6/guides/training-agents.mdx b/docs/v6/guides/training-agents.mdx index 11f1abf13..de055b4a7 100644 --- a/docs/v6/guides/training-agents.mdx +++ b/docs/v6/guides/training-agents.mdx @@ -93,7 +93,7 @@ One job spans the session; each step appends a batch and trains on it: - **Open the job** with `group=8` - 8 rollouts per task, so the rewards are comparable (next). - **Roll out** the batch, the same eval as the [previous guide](/v6/guides/running-an-eval). The - [runtime](/v6/reference/runtime) sets where it runs; swap `LocalRuntime` for `HUDRuntime()` unchanged. + [runtime](/v6/reference/runtime) sets where it runs; swap `SubprocessRuntime` for `HUDRuntime()` unchanged. - **Nudge** with `trainer.step` - the one line that learns. It scores each rollout against its group, shifts the weights, then **promotes** them so the gateway serves the new ones at once. @@ -106,7 +106,7 @@ One job spans the session; each step appends a batch and trains on it: ```python import asyncio -from hud import TrainingClient, Taskset, LocalRuntime +from hud import TrainingClient, Taskset, SubprocessRuntime from hud.agents import create_agent from hud.eval import Job @@ -122,7 +122,7 @@ async def main(): session = await Job.start(MODEL, group=8) # one job spans the session for step in range(10): start = len(session.runs) - await taskset.run(agent, runtime=LocalRuntime("env.py"), job=session) + await taskset.run(agent, runtime=SubprocessRuntime("env.py"), job=session) batch = session.runs[start:] # this step's rollouts await trainer.step(batch, learning_rate=1e-5, group_size=8) # nudge + promote print(f"step {step} reward {sum(r.reward for r in batch) / len(batch):.2f}") diff --git a/docs/v6/reference/agents.mdx b/docs/v6/reference/agents.mdx index 1f32b2961..3c91734ad 100644 --- a/docs/v6/reference/agents.mdx +++ b/docs/v6/reference/agents.mdx @@ -105,11 +105,11 @@ with a [runtime](/v6/reference/runtime): ```python from hud.agents import create_agent -from hud.eval import LocalRuntime, Taskset +from hud.eval import SubprocessRuntime, Taskset agent = create_agent("claude-sonnet-4-5") taskset = Taskset.from_file("tasks.py") # scaffolded tasks.py exports a list of tasks -job = await taskset.run(agent, runtime=LocalRuntime("env.py")) +job = await taskset.run(agent, runtime=SubprocessRuntime("env.py")) print(job.reward) ``` diff --git a/docs/v6/reference/environment.mdx b/docs/v6/reference/environment.mdx index 6a38f8c93..ad1a60418 100644 --- a/docs/v6/reference/environment.mdx +++ b/docs/v6/reference/environment.mdx @@ -59,7 +59,7 @@ and [Tasks & Tasksets](/v6/reference/tasks). You rarely serve by hand - `hud eval`, [`task.run()`](/v6/reference/tasks), and `Taskset.run()` bring the environment up for you, and the [runtime](/v6/reference/runtime) you pass decides where. Serving itself belongs to `hud.environment.server`, the entry point every substrate runs: a -[`LocalRuntime`](/v6/reference/runtime#localruntime) child process, a container CMD, or `hud serve`. +[`SubprocessRuntime`](/v6/reference/runtime#subprocessruntime) child process, a container CMD, or `hud serve`. | Function | Description | |----------|-------------| diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index 1db44baa1..cc39495fa 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -9,16 +9,18 @@ A **runtime** chooses where each rollout's environment runs. You pass it to `tas runtime changes. ```python -from hud import LocalRuntime +from hud import LocalRuntime, SubprocessRuntime -await taskset.run(agent, runtime=LocalRuntime("env.py")) # serve env.py locally, run here +await taskset.run(agent, runtime=LocalRuntime(env)) # serve a live env in-process +await taskset.run(agent, runtime=SubprocessRuntime("env.py")) # serve env.py in a child process ``` ## Built-in runtimes | Runtime | Where the env runs | When to reach for it | |---------|--------------------|----------------------| -| `LocalRuntime("env.py")` | A child process from your source | Fastest iteration; local development | +| `LocalRuntime(env)` | This process, serving live `Environment` objects | Fastest iteration; envs materialized in code | +| `SubprocessRuntime("env.py")` | A child process from your source | Local development with process isolation | | `DockerRuntime("my-env")` | A fresh local container per rollout | Reproducibility and parity with production | | `ModalRuntime("my-env")` | A fresh [Modal](https://modal.com/) sandbox per rollout | Cloud scale, no infra to manage | | `DaytonaRuntime("my-env")` | A fresh [Daytona](https://www.daytona.io/) sandbox per rollout | Cloud scale on Daytona | @@ -26,14 +28,14 @@ await taskset.run(agent, runtime=LocalRuntime("env.py")) # serve env.py locall | `HUDRuntime()` | A HUD-hosted env, leased by name and tunneled | Local agent loop against a deployed env | | `HostedRuntime()` | The whole rollout on a HUD-leased box | Agent and env run together off your machine | -Most runtimes are on the top-level package (`from hud import LocalRuntime, DockerRuntime, HUDRuntime, -HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.eval`. +Most runtimes are on the top-level package (`from hud import LocalRuntime, SubprocessRuntime, +DockerRuntime, HUDRuntime, HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from +`hud.eval`. -**Omit `runtime=` and it's inferred** from each task's `_source`, the file its template was defined -in. When every task shares one `_source`, that source is served locally as `LocalRuntime(source)`; -otherwise (mixed sources, or rows loaded from a file or the platform with no source) it falls back to -`HUDRuntime()`. Pass a runtime explicitly the moment you want something else. +**Omit `runtime=`** and the run defaults to `HUDRuntime()` - the deployed env leased by each row's +`env` name. Tasks are pure data; a placement is always supplied at run time, never recovered from +the rows. To deploy an environment to the platform and run against it, see @@ -65,7 +67,7 @@ RuntimeConfig( Support differs per runtime: `DockerRuntime`, `ModalRuntime`, and `DaytonaRuntime` accept it (Docker ignores `limits`; Daytona ignores `run_timeout_s` and resource overrides when booting from a snapshot). -`LocalRuntime` and `HUDRuntime` reject a per-task `runtime_config`. +`LocalRuntime`, `SubprocessRuntime`, and `HUDRuntime` reject a per-task `runtime_config`. ## Runtime directory @@ -74,7 +76,25 @@ The constructor for each built-in runtime: ### `LocalRuntime` ```python -LocalRuntime(path, *, env=None, ready_timeout=120.0) +LocalRuntime(envs) +``` + +- **`envs`** - a live `Environment`, or a mapping of env name to `Environment` / zero-arg factory + for mixed-env tasksets. Rows join by `task.env` name. + +An `Environment` *instance* is a shared substrate: its daemons start on the first acquisition and +stop after the last, every rollout shares the env's capabilities and state, and each rollout gets its +own control channel so concurrent runs never collide. A *factory* is fresh per acquisition - built, +served, and stopped around each rollout - for envs whose state a rollout mutates. + +Serving is in-process on a loopback port through the same control channel as every placement; only +isolation differs. Env hooks share this process's event loop, so keep envs async - or use +`SubprocessRuntime` / `DockerRuntime` when the env should not share the orchestrator's fate. + +### `SubprocessRuntime` + +```python +SubprocessRuntime(path, *, env=None, ready_timeout=120.0) ``` - **`path`** - `.py` file (or directory) that declares the env. The child's working directory is the source's directory, so sibling imports and relative data paths resolve. diff --git a/docs/v6/reference/tasks.mdx b/docs/v6/reference/tasks.mdx index 0a764ed50..ef364f47a 100644 --- a/docs/v6/reference/tasks.mdx +++ b/docs/v6/reference/tasks.mdx @@ -145,9 +145,9 @@ collections: `task.run(...)` is the same call over a taskset of one, with identical semantics. ```python -from hud import LocalRuntime +from hud import SubprocessRuntime -job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=8, max_concurrent=10) +job = await ts.run(agent, runtime=SubprocessRuntime("env.py"), group=8, max_concurrent=10) ``` | Parameter | Type | Description | diff --git a/docs/v6/reference/types.mdx b/docs/v6/reference/types.mdx index 21c77c91c..600bc3b33 100644 --- a/docs/v6/reference/types.mdx +++ b/docs/v6/reference/types.mdx @@ -67,7 +67,7 @@ run reports under a job, so even a single `task.run` returns a job of one. You g | `results` | `dict[str, list[Run]]` | Property: runs grouped by task slug - the alignment-safe alternative to `zip(tasks, runs)`, list-valued since `group > 1` gives several runs per task. | ```python -job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=4) +job = await ts.run(agent, runtime=SubprocessRuntime("env.py"), group=4) print(job.reward) # mean across every run ``` diff --git a/docs/v6/start/overview.mdx b/docs/v6/start/overview.mdx index 35ef95bbd..f8ef66a10 100644 --- a/docs/v6/start/overview.mdx +++ b/docs/v6/start/overview.mdx @@ -155,9 +155,10 @@ hud eval env.py claude --runtime hud # same env, executed on HUD's hosted inf [runtime](/v6/reference/runtime) and run a taskset against it: ```python -from hud.eval import LocalRuntime, DockerRuntime, ModalRuntime, HUDRuntime +from hud.eval import LocalRuntime, SubprocessRuntime, DockerRuntime, ModalRuntime, HUDRuntime -LocalRuntime("env.py") # local child process - fastest iteration +LocalRuntime(env) # this process - live env objects +SubprocessRuntime("env.py") # local child process serving a source file DockerRuntime("my-env") # a fresh container per rollout ModalRuntime("my-env") # a Modal cloud sandbox per rollout HUDRuntime() # HUD's hosted infra (after `hud deploy`) @@ -199,12 +200,12 @@ You can run this programmatically: ```python from hud.agents import create_agent -from hud.eval import LocalRuntime +from hud.eval import SubprocessRuntime from tasks import TASKS agent = create_agent("claude-sonnet-4-5") # routed through the HUD gateway -job = await TASKS.run(agent, runtime=LocalRuntime("env.py")) # start the run +job = await TASKS.run(agent, runtime=SubprocessRuntime("env.py")) # start the run print(job.reward) ``` {/* diff --git a/hud/__init__.py b/hud/__init__.py index 268724cae..76f4b5e54 100644 --- a/hud/__init__.py +++ b/hud/__init__.py @@ -24,6 +24,7 @@ RuntimeGPU, RuntimeLimits, RuntimeResources, + SubprocessRuntime, SyncPlan, Task, Taskset, @@ -49,6 +50,7 @@ "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "SubprocessRuntime", "SyncPlan", "Task", "Taskset", diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 39afd6edf..0a071b53b 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -708,7 +708,7 @@ def _python_defines_environment(path: Path) -> bool: def _spawn_target(source: Path) -> Path: - """The path the ``LocalRuntime`` provider serves. + """The path the ``SubprocessRuntime`` provider serves. Directories and env-defining ``.py`` files are served as-is. Task-only sources (``tasks.py`` importing from ``env.py``) resolve to a sibling @@ -736,7 +736,7 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any: ``--remote`` submits every rollout for platform-hosted execution; a ``tcp://`` url attaches to an env served elsewhere. """ - from hud.eval import HostedRuntime, HUDRuntime, LocalRuntime, Runtime + from hud.eval import HostedRuntime, HUDRuntime, Runtime, SubprocessRuntime if cfg.remote: require_api_key("run remote hosted evals") @@ -744,7 +744,7 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any: if cfg.runtime == "local": if source_path is None: raise ValueError("local placement requires a local source path") - return LocalRuntime(_spawn_target(source_path)) + return SubprocessRuntime(_spawn_target(source_path)) if cfg.runtime == "hud": require_api_key("run HUD runtime tunnel evals") return HUDRuntime() diff --git a/hud/cli/task.py b/hud/cli/task.py index 59a0637e4..3e15470b2 100644 --- a/hud/cli/task.py +++ b/hud/cli/task.py @@ -94,7 +94,7 @@ def _resolve( """ from contextlib import nullcontext - from hud.eval.runtime import LocalRuntime, Runtime + from hud.eval.runtime import Runtime, SubprocessRuntime attach = url if attach is None and source is None: @@ -118,7 +118,7 @@ def _resolve( hud_console.error(f"No task matching {task!r} (available: {available})") raise typer.Exit(1) selected = matches[0] - placement = LocalRuntime(_spawn_target(source or "."))(selected) + placement = SubprocessRuntime(_spawn_target(source or "."))(selected) return selected.id, args or selected.args, placement diff --git a/hud/cli/templates.py b/hud/cli/templates.py index 5be236857..28c560a5e 100644 --- a/hud/cli/templates.py +++ b/hud/cli/templates.py @@ -87,10 +87,10 @@ async def test(): agent = ClaudeAgent() - # Calling a task binds a runnable Task; ``runtime=LocalRuntime(__file__)`` serves this - # file in a child process and runs the task against it over the wire. + # Calling a task binds a runnable Task; ``runtime=LocalRuntime(env)`` serves the + # live env in this process and runs the task against it over the wire. task = count(sentence="Strawberry world", letter="r") - job = await task.run(agent, runtime=LocalRuntime(__file__)) + job = await task.run(agent, runtime=LocalRuntime(env)) print("reward:", job.reward) diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index 94274f173..0950965b6 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -5,7 +5,8 @@ :mod:`~hud.environment.server` is the serving entry point substrates run. How a substrate comes up — placement — belongs to the eval engine: see :mod:`hud.eval.runtime` (:class:`~hud.eval.runtime.Runtime`, the ``Provider`` -contract, ``LocalRuntime``, ``DockerRuntime``, ``HUDRuntime``). +contract, ``LocalRuntime``, ``SubprocessRuntime``, ``DockerRuntime``, +``HUDRuntime``). The env-side robot runtime (bridges, action providers, sim runners, contract tooling, recording glue) lives in :mod:`hud.environment.robot`; import it diff --git a/hud/environment/env.py b/hud/environment/env.py index 07e1afd80..960de529d 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -76,7 +76,7 @@ class _TaskFactory(Generic[P]): binds a runnable :class:`~hud.eval.Task`:: task = fix_bug(difficulty=3) # -> Task - job = await task.run(agent, runtime=LocalRuntime("env.py")) + job = await task.run(agent, runtime=LocalRuntime(env)) """ def __init__( @@ -120,13 +120,7 @@ def __call__(self, *args: P.args, **kwargs: P.kwargs) -> EvalTask: from hud.eval.task import Task bound = self.sig.bind(*args, **kwargs) - task = Task(env=self.env.name, id=self.id, args=dict(bound.arguments)) - # Record where this template was defined so ``task.run()`` can default to - # serving that source locally (in-process only; never crosses the wire). - source = inspect.getsourcefile(self.func) - if source is not None: - task._source = source - return task + return Task(env=self.env.name, id=self.id, args=dict(bound.arguments)) class Environment(LegacyEnvMixin): diff --git a/hud/environment/server.py b/hud/environment/server.py index e18f64a43..f4e264c22 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -6,7 +6,7 @@ (:func:`bind`), and the full serving lifecycle (:func:`serve`) — backing daemons up, control channel bound (announcing the port on stdout as ``HUD_SERVE_PORT=``), daemons down. Every substrate shape runs it: the -:class:`~hud.eval.runtime.LocalRuntime` child process, a container CMD, and +:class:`~hud.eval.runtime.SubprocessRuntime` child process, a container CMD, and ``hud serve``. """ diff --git a/hud/eval/__init__.py b/hud/eval/__init__.py index 83f7970ab..43765fc6c 100644 --- a/hud/eval/__init__.py +++ b/hud/eval/__init__.py @@ -13,16 +13,17 @@ exception: calling an ``@env.template`` declaration constructs the eval ``Task`` row.) -Placement is passed at execution time (see :mod:`.runtime`): ``LocalRuntime`` a -local source, ``DockerRuntime`` an image, ``Runtime(url)`` an env served -elsewhere, ``HUDRuntime`` a HUD runtime tunnel, or ``HostedRuntime`` to run the -whole rollout remotely on the platform:: +Placement is passed at execution time (see :mod:`.runtime`): ``LocalRuntime`` +live envs in this process, ``SubprocessRuntime`` a local source, +``DockerRuntime`` an image, ``Runtime(url)`` an env served elsewhere, +``HUDRuntime`` a HUD runtime tunnel, or ``HostedRuntime`` to run the whole +rollout remotely on the platform:: - from hud.eval import LocalRuntime, Taskset + from hud.eval import LocalRuntime, SubprocessRuntime, Taskset - job = await my_task(a=1).run(agent, runtime=LocalRuntime("env.py")) + job = await my_task(a=1).run(agent, runtime=LocalRuntime(env)) job = await Taskset("demo", [my_task(d) for d in range(5)]).run( - agent, runtime=LocalRuntime("env.py"), group=8 + agent, runtime=SubprocessRuntime("env.py"), group=8 ) """ @@ -46,6 +47,7 @@ RuntimeGPU, RuntimeLimits, RuntimeResources, + SubprocessRuntime, ) from .sync import SyncPlan from .task import Task @@ -68,6 +70,7 @@ "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "SubprocessRuntime", "SyncPlan", "Task", "Taskset", diff --git a/hud/eval/chat.py b/hud/eval/chat.py index 836f41713..1e0e60730 100644 --- a/hud/eval/chat.py +++ b/hud/eval/chat.py @@ -96,7 +96,8 @@ def __init__( (stateless per run, e.g. ``create_agent("claude-sonnet-4-5")``). runtime: The env placement each turn's rollout runs against — a :class:`~hud.eval.runtime.Provider` such as - ``LocalRuntime("env.py")`` or ``Runtime("tcp://...")``. Chat is + ``LocalRuntime(env)``, ``SubprocessRuntime("env.py")``, or + ``Runtime("tcp://...")``. Chat is interactive and local: it drives the agent loop in this process, so hosted placement does not apply. """ @@ -133,7 +134,7 @@ async def send(self, message: MessageContent) -> Trace: if self._runtime is None: raise RuntimeError( "Chat needs a runtime to converse against — pass an env placement, " - 'e.g. runtime=Runtime("tcp://...") or runtime=LocalRuntime("env.py").' + 'e.g. runtime=LocalRuntime(env) or runtime=Runtime("tcp://...").' ) if self.job is None: # one job spans the whole conversation self.job = await Job.start(self._task.id) diff --git a/hud/eval/run.py b/hud/eval/run.py index 5949786bd..2df207d06 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -5,7 +5,7 @@ loopback, a container, a cloud sandbox), starts the task, drives the agent, grades, and tears down, filling a :class:`Run` along the way:: - run = await rollout(task, agent, runtime=LocalRuntime("env.py")) + run = await rollout(task, agent, runtime=SubprocessRuntime("env.py")) It is the *client-here* path: the agent loop runs in this process against a :class:`~hud.eval.runtime.Provider`'s channel. The same driver runs on the diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index eea8bba34..0015f9090 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -6,8 +6,12 @@ transparent, so "co-located" (loopback) and "split" (agent here, env elsewhere) are the same code, differing only in the url. -- :class:`LocalRuntime` — runs a subprocess serving the row's env from a ``.py`` - source (the path is always given, never recovered from a live object). +- :class:`LocalRuntime` — serves live :class:`Environment` objects in-process + (rows join by env name; instances are shared substrates, factories fresh + per acquisition). +- :class:`SubprocessRuntime` — runs a child process serving the row's env from + a ``.py`` source (the path is always given, never recovered from a live + object). - :class:`DockerRuntime` — ``docker run``s an image whose CMD serves the channel. - ``Runtime(url)`` — the ``nullcontext`` of providers: yields itself, a *borrowed, shared* substrate provisioned elsewhere (env served anywhere — @@ -32,6 +36,7 @@ import sys import uuid from collections import deque +from collections.abc import Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from dataclasses import dataclass, field from pathlib import Path @@ -49,7 +54,7 @@ from .run import Grade, Run, rollout if TYPE_CHECKING: - from collections.abc import AsyncIterator, Mapping, Sequence + from collections.abc import AsyncIterator, Callable, Sequence from hud.agents.base import Agent from hud.environment.env import Environment @@ -154,7 +159,101 @@ def _modal_image_from_uri(modal: Any, image_uri: str) -> Any: class LocalRuntime: - """The local provider: serve the placed row's env from *path* in a child process. + """The in-process provider: serve live :class:`Environment` objects from here. + + The placement whose right-hand side is objects you already hold — a single + env, or a mapping of env name to env / zero-arg factory for a mixed-env + taskset. Rows join by ``task.env`` name, like every placement:: + + job = await taskset.run(agent, runtime=LocalRuntime(env)) + job = await taskset.run(agent, runtime=LocalRuntime({"tb-g1": make_g1})) + + An :class:`Environment` *instance* is a shared substrate: its daemons + start on first acquisition and stop after the last, and every rollout + placed on it shares the env's capabilities and state — but each gets its + own control channel (a bound channel holds at most one suspended task, so + concurrent runs never collide). A *factory* is fresh per acquisition — + built, served, and stopped around each rollout — for envs whose state a + rollout mutates. + + Serving is in-process on a loopback port, through the same control channel + as any placement — only isolation differs: env hooks run in this process + and share its event loop, so blocking env code stalls concurrent rollouts. + Use :class:`SubprocessRuntime` or :class:`DockerRuntime` when the env + should not share the orchestrator's fate. + """ + + def __init__( + self, + envs: Environment | Mapping[str, Environment | Callable[[], Environment]], + ) -> None: + if isinstance(envs, (str, Path)): + raise TypeError( + "LocalRuntime serves live Environment objects; " + "use SubprocessRuntime(path) to serve a source file." + ) + from hud.environment.env import Environment as _Environment + + if isinstance(envs, Mapping): + self._envs: dict[str, _Environment | Callable[[], _Environment]] = dict(envs) + elif isinstance(envs, _Environment): + self._envs = {envs.name: envs} + else: + # A factory has no name until called, so the join key must be + # explicit: pass factories as {env_name: factory}. + raise TypeError( + f"LocalRuntime: expected an Environment or a mapping of " + f"env name -> Environment/factory; got {envs!r}" + ) + if not self._envs: + raise ValueError("LocalRuntime: no environments given") + # Shared-instance daemon refcounts, keyed by env name: start on first + # acquisition, stop after the last. + self._leases: dict[str, int] = {} + self._lock = asyncio.Lock() + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + from hud.environment.env import Environment as _Environment + + if task.runtime_config is not None: + raise ValueError("LocalRuntime does not support task runtime_config") + entry = self._envs.get(task.env) + if entry is None: + raise KeyError( + f"LocalRuntime has no environment named {task.env!r} (has: {sorted(self._envs)})" + ) + if isinstance(entry, _Environment): + async with self._acquire_shared(task.env, entry) as runtime: + yield runtime + return + env = entry() + if not isinstance(env, _Environment): + raise TypeError( + f"LocalRuntime factory for {task.env!r} returned {env!r}, not an Environment" + ) + async with _local(env) as runtime: + yield runtime + + @asynccontextmanager + async def _acquire_shared(self, name: str, env: Environment) -> AsyncIterator[Runtime]: + async with self._lock: + if self._leases.get(name, 0) == 0: + await env.start() + self._leases[name] = self._leases.get(name, 0) + 1 + try: + async with _bind_channel(env) as runtime: + yield runtime + finally: + async with self._lock: + self._leases[name] -= 1 + if self._leases[name] == 0: + del self._leases[name] + await env.stop() + + +class SubprocessRuntime: + """The child-process provider: serve the placed row's env from *path*. Each acquisition runs ``python -m hud.environment.server --env name`` — the same serving entry point a container CMD runs — on an @@ -167,8 +266,8 @@ class LocalRuntime: The child's working directory is the source's directory, so sibling imports and relative data paths resolve; ``@env.initialize`` daemons start in the child and die with it. Because the source is re-imported in the - child, a script spawning itself (``LocalRuntime(__file__)``) must keep top-level - run calls under ``if __name__ == "__main__":``. + child, a script spawning itself (``SubprocessRuntime(__file__)``) must keep + top-level run calls under ``if __name__ == "__main__":``. """ def __init__( @@ -185,9 +284,9 @@ def __init__( @asynccontextmanager async def __call__(self, task: Task) -> AsyncIterator[Runtime]: if task.runtime_config is not None: - raise ValueError("LocalRuntime does not support task runtime_config") + raise ValueError("SubprocessRuntime does not support task runtime_config") if not self.source.exists(): - raise FileNotFoundError(f"LocalRuntime: source not found: {self.source}") + raise FileNotFoundError(f"SubprocessRuntime: source not found: {self.source}") cmd = [sys.executable, "-m", "hud.environment.server", str(self.source)] cmd += ["--env", self.env or task.env] proc = await create_process_group_exec( @@ -603,18 +702,15 @@ async def _docker(*args: str, check: bool = True) -> tuple[str, str]: @asynccontextmanager -async def _local(env: Environment) -> AsyncIterator[Runtime]: - """Substrate-side serving: a live env owned by *this* process, as a runtime. +async def _bind_channel(env: Environment) -> AsyncIterator[Runtime]: + """Bind one control channel over an already-started env, as a runtime. - Not a placement the engine offers (the orchestrator never serves an env - in-process), so deliberately not a ``Provider`` — it serves a live object, - not a placed row. Code already running *inside* a placed substrate adapts - it (``AgentTool`` sub-rollouts: ``runtime=lambda _: _local(env)``); test - harnesses enter it directly. + Each bound channel holds at most one suspended task, so concurrent + rollouts on a shared env each get their own channel (see + ``LocalRuntime``); the env's daemon lifecycle is the caller's concern. """ from hud.environment.server import bind - await env.start() server = await bind(env, "127.0.0.1", 0) host, port = server.sockets[0].getsockname()[:2] serve_task = asyncio.create_task(server.serve_forever()) @@ -627,6 +723,22 @@ async def _local(env: Environment) -> AsyncIterator[Runtime]: server.close() with contextlib.suppress(Exception): await server.wait_closed() + + +@asynccontextmanager +async def _local(env: Environment) -> AsyncIterator[Runtime]: + """Substrate-side serving: a live env owned by *this* process, as a runtime. + + One env lifecycle (start → serve → stop) around one bound channel — the + single-use form ``LocalRuntime`` builds on. Code already running *inside* + a placed substrate adapts it (``AgentTool`` sub-rollouts: + ``runtime=lambda _: _local(env)``); test harnesses enter it directly. + """ + await env.start() + try: + async with _bind_channel(env) as runtime: + yield runtime + finally: await env.stop() @@ -1011,4 +1123,5 @@ async def ws_to_tcp() -> None: "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "SubprocessRuntime", ] diff --git a/hud/eval/task.py b/hud/eval/task.py index ab3363ae2..d942f462c 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -24,7 +24,7 @@ import json from typing import TYPE_CHECKING, Any -from pydantic import BaseModel, Field, PrivateAttr +from pydantic import BaseModel, Field from .runtime import RuntimeConfig @@ -40,8 +40,8 @@ class Task(BaseModel): Pure data — holds no execution state, so one ``Task`` can drive many concurrent rollouts. ``run`` it for a graded :class:`~hud.eval.job.Job`; - placement comes from ``runtime=`` (a provider), else the source the task was - minted from (local), else the HUD runtime tunnel by ``env`` name. + placement comes from ``runtime=`` (a provider), else the HUD runtime + tunnel by ``env`` name. """ env: str = Field(min_length=1) @@ -57,12 +57,6 @@ class Task(BaseModel): #: supported subset into their native launch shape or reject it. runtime_config: RuntimeConfig | None = None - #: In-process only: the source file the template was defined in, captured - #: when a template factory mints the task. Lets ``run`` default to serving - #: that source locally. Excluded from the wire (a row loaded from JSON has - #: none, and falls back to HUD runtime tunnel placement). - _source: str | None = PrivateAttr(default=None) - def default_slug(self) -> str: """A stable slug from the task id, disambiguated by an args hash when present.""" if not self.args: @@ -90,8 +84,8 @@ async def run( open ``job`` from :meth:`Job.start` to accumulate into), ``group`` repeats sharing a group_id, ``max_concurrent`` capping parallelism — over a taskset of one. ``runtime`` is the placement; left unset it - serves the task's source locally when minted in-process, else falls - back to the HUD runtime tunnel by ``env`` name. + falls back to the HUD runtime tunnel by ``env`` name. To run against + a live env in this process, pass ``runtime=LocalRuntime(env)``. """ from .taskset import Taskset # circular: taskset -> sync -> task diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index a7f7e9cea..b9a8b296f 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -5,7 +5,7 @@ :mod:`hud.eval.job`; platform persistence in :mod:`hud.eval.sync`:: job = await Taskset("bugs", [fix_bug(difficulty=d) for d in range(5)]).run( - agent, runtime=LocalRuntime("env.py") + agent, runtime=SubprocessRuntime("env.py") ) """ @@ -23,7 +23,7 @@ from .job import Job, job_enter from .run import rollout -from .runtime import HostedRuntime, HUDRuntime, LocalRuntime +from .runtime import HostedRuntime, HUDRuntime from .sync import fetch_taskset_tasks, resolve_taskset_id if TYPE_CHECKING: @@ -264,13 +264,9 @@ async def run( # Placement is chosen once for the batch: HostedRuntime delegates the # whole rollout to the platform, anything else is a Provider driven - # locally by rollout(). - # No runtime: serve the tasks' shared source locally if they were minted - # in-process from one file (the common authoring case); otherwise (mixed - # or wire-loaded rows with no source) default to the HUD runtime tunnel. - if runtime is None: - sources = {t._source for t in task_list if t._source is not None} - runtime = LocalRuntime(next(iter(sources))) if len(sources) == 1 else None + # locally by rollout(). No runtime defaults to the HUD runtime tunnel + # by env name; live envs in this process are a placement too + # (``runtime=LocalRuntime(env)``), never recovered from the rows. placement = runtime if runtime is not None else HUDRuntime() sem = asyncio.Semaphore(max_concurrent) if max_concurrent else None diff --git a/hud/eval/tests/test_chat.py b/hud/eval/tests/test_chat.py index 68b6bdaf0..47220ccfb 100644 --- a/hud/eval/tests/test_chat.py +++ b/hud/eval/tests/test_chat.py @@ -1,6 +1,6 @@ """``Chat`` — multi-turn conversation runner over a task. -Turn tests place each turn's rollout with ``runtime=LocalRuntime(env_file)`` — a pure-data +Turn tests place each turn's rollout with ``runtime=SubprocessRuntime(env_file)`` — a pure-data ``Task`` row against a chat-style env served from a child process. """ @@ -13,7 +13,7 @@ from mcp.types import TextContent from hud.agents.base import Agent -from hud.eval import LocalRuntime, Task +from hud.eval import SubprocessRuntime, Task from hud.eval.chat import Chat, _content_to_blocks if TYPE_CHECKING: @@ -89,7 +89,7 @@ class TestSend: async def test_send_runs_a_turn_and_stores_prompt_message_format( self, chat_env_file: Path ) -> None: - chat = Chat(_chat_task(), _EchoAgent(), runtime=LocalRuntime(chat_env_file)) + chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file)) trace = await chat.send("hello") @@ -107,7 +107,7 @@ async def test_send_runs_a_turn_and_stores_prompt_message_format( assert assistant_msg["content"]["text"] == "echo:hello" async def test_one_job_spans_the_conversation(self, chat_env_file: Path) -> None: - chat = Chat(_chat_task(), _EchoAgent(), runtime=LocalRuntime(chat_env_file)) + chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file)) await chat.send("hello") await chat.send("again") @@ -125,7 +125,7 @@ class _Boom(Agent): async def __call__(self, run: Any) -> None: raise RuntimeError("agent exploded") - chat = Chat(_chat_task(), _Boom(), runtime=LocalRuntime(chat_env_file)) + chat = Chat(_chat_task(), _Boom(), runtime=SubprocessRuntime(chat_env_file)) with pytest.raises(RuntimeError, match="agent exploded"): await chat.send("hello") diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py new file mode 100644 index 000000000..9b676b417 --- /dev/null +++ b/hud/eval/tests/test_local_runtime.py @@ -0,0 +1,180 @@ +"""LocalRuntime: the in-process placement over live ``Environment`` objects. + +Rows join by env name like every placement; an ``Environment`` instance is a +shared substrate (daemons started once, refcounted across acquisitions, one +control channel per acquisition), a factory in a mapping is fresh per +acquisition. Everything still crosses the control channel — these tests drive +the real rollout engine against envs that only exist in this process. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime +from typing import Any, cast + +import pytest + +from hud.agents.base import Agent +from hud.environment import Environment +from hud.eval import LocalRuntime, Task, Taskset +from hud.eval.run import rollout + + +def _sums_env(name: str = "sums") -> Environment: + env = Environment(name) + + @env.template(id="add") + async def add(a: int, b: int) -> AsyncGenerator[Any, Any]: + answer = yield f"add:{a}:{b}" + yield 1.0 if answer == str(a + b) else 0.0 + + return env + + +class _FnAgent(Agent): + """Stateless agent: answers each run by applying ``fn`` to ``run.prompt``.""" + + def __init__(self, fn: Any) -> None: + self._fn = fn + + async def __call__(self, run: Any) -> None: + run.trace.content = self._fn(run.prompt) + + +def _solve_add(prompt: str) -> str: + _, a, b = prompt.split(":") + return str(int(a) + int(b)) + + +async def test_live_env_rollout_end_to_end() -> None: + run = await rollout( + Task(env="sums", id="add", args={"a": 2, "b": 3}), + _FnAgent(_solve_add), + runtime=LocalRuntime(_sums_env()), + ) + + assert run.reward == 1.0 + assert run.trace_id + + +async def test_shared_instance_serves_once_across_concurrent_acquisitions() -> None: + env = _sums_env() + starts, stops = [], [] + + @env.initialize + async def _up() -> None: + starts.append(1) + + @env.shutdown + async def _down() -> None: + stops.append(1) + + provider = LocalRuntime(env) + task = Task(env="sums", id="add") + release = asyncio.Event() + all_acquired = asyncio.Event() + urls: list[str] = [] + + async def _hold() -> None: + async with provider(task) as runtime: + urls.append(runtime.url) + if len(urls) == 3: + all_acquired.set() + await release.wait() + + holders = [asyncio.create_task(_hold()) for _ in range(3)] + await all_acquired.wait() + # One shared env (daemons started once), but one channel per acquisition + # so concurrent task lifecycles never collide. + assert len(set(urls)) == 3 + assert starts == [1] + assert stops == [] + + release.set() + await asyncio.gather(*holders) + assert stops == [1] + + +async def test_shared_env_runs_grouped_taskset() -> None: + taskset = Taskset( + "sums", + [Task(env="sums", id="add", args={"a": a, "b": a + 1}, slug=f"add-{a}") for a in range(3)], + ) + + job = await taskset.run( + _FnAgent(_solve_add), + runtime=LocalRuntime(_sums_env()), + group=2, + max_concurrent=3, + ) + + assert len(job.runs) == 6 + assert all(run.reward == 1.0 for run in job.runs) + + +async def test_factory_builds_fresh_env_per_acquisition() -> None: + built: list[Environment] = [] + + def factory() -> Environment: + env = _sums_env() + built.append(env) + return env + + task = Task(env="sums", id="add", args={"a": 1, "b": 2}) + job = await task.run( + _FnAgent(_solve_add), + runtime=LocalRuntime({"sums": factory}), + group=3, + ) + + assert all(run.reward == 1.0 for run in job.runs) + assert len(built) == 3 + + +async def test_mapping_joins_rows_by_env_name() -> None: + doubles = Environment("doubles") + + @doubles.template(id="double") + async def double(n: int) -> AsyncGenerator[Any, Any]: + answer = yield f"double:{n}" + yield 1.0 if answer == str(2 * n) else 0.0 + + def _solve(prompt: str) -> str: + kind, *parts = prompt.split(":") + if kind == "double": + return str(2 * int(parts[0])) + return str(int(parts[0]) + int(parts[1])) + + taskset = Taskset( + "mixed", + [ + Task(env="sums", id="add", args={"a": 4, "b": 5}), + Task(env="doubles", id="double", args={"n": 7}), + ], + ) + + job = await taskset.run( + _FnAgent(_solve), + runtime=LocalRuntime({"sums": _sums_env(), "doubles": doubles}), + ) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + + +async def test_unknown_env_name_fails_loudly() -> None: + provider = LocalRuntime(_sums_env()) + + with pytest.raises(KeyError, match="no environment named 'other'"): + async with provider(Task(env="other", id="add")): + pass + + +def test_rejects_path_argument_pointing_at_subprocess_runtime() -> None: + with pytest.raises(TypeError, match="SubprocessRuntime"): + LocalRuntime(cast("Any", "env.py")) + + +def test_rejects_bare_factory_without_a_name() -> None: + with pytest.raises(TypeError, match="mapping"): + LocalRuntime(cast("Any", _sums_env)) diff --git a/hud/eval/tests/test_rollout.py b/hud/eval/tests/test_rollout.py index 0d8bb5166..3473d5a1c 100644 --- a/hud/eval/tests/test_rollout.py +++ b/hud/eval/tests/test_rollout.py @@ -1,7 +1,7 @@ """The rollout engine: ``rollout(task, agent)`` and its schedulers. These drive the engine end-to-end through the real placement path: a pure-data -``Task`` row plus ``runtime=LocalRuntime(env_file)`` — a child process serves the env, the +``Task`` row plus ``runtime=SubprocessRuntime(env_file)`` — a child process serves the env, the engine connects over the wire, the agent answers, grading comes back. The engine contract is a graded :class:`Run` with a trace id (always under a job — there are no standalone traces), and failure isolation that never raises: a @@ -31,7 +31,7 @@ from hud.agents.openai_compatible import OpenAIChatAgent from hud.agents.types import OpenAIChatConfig from hud.environment import Environment -from hud.eval import Job, LocalRuntime, Task, Taskset +from hud.eval import Job, SubprocessRuntime, Task, Taskset from hud.eval.run import Run, rollout from hud.eval.runtime import _local @@ -155,7 +155,7 @@ async def _wait_for_pid_inactive(pid: int, max_wait: float = 2.0) -> bool: async def test_rollout_returns_graded_run_with_trace_id(env_file: Path) -> None: - run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=LocalRuntime(env_file)) + run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file)) assert run.reward == 1.0 assert run.trace.content == "5" @@ -249,7 +249,7 @@ async def start_child(): try: with pytest.raises(RuntimeError, match="startup boom"): - async with LocalRuntime(env_file, ready_timeout=2.0)(Task(env="leaky", id="noop")): + async with SubprocessRuntime(env_file, ready_timeout=2.0)(Task(env="leaky", id="noop")): pass pid = int(pid_file.read_text()) assert await _wait_for_pid_inactive(pid) @@ -263,7 +263,7 @@ async def test_mid_run_failure_keeps_the_real_run_and_its_evidence(env_file: Pat def boom(prompt: str) -> str: raise RuntimeError("agent exploded") - run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=LocalRuntime(env_file)) + run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=SubprocessRuntime(env_file)) assert run.trace.is_error assert "agent exploded" in (run.trace.error or "") @@ -291,7 +291,7 @@ async def test_mid_run_failure_still_grades_best_effort(env_file: Path) -> None: # The agent answers correctly, then fails. The env is still alive, so the # run is graded best-effort: the reward is captured even though it errored. run = await rollout( - _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=LocalRuntime(env_file) + _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=SubprocessRuntime(env_file) ) assert run.trace.is_error @@ -358,7 +358,7 @@ def placer(task: TaskRow) -> Any: # The scheduler half of placement: the row is the request, so a # provider can size/route each substrate per task. placed.append(f"{task.env}/{task.id}:{task.args['a']}") - return LocalRuntime(env_file)(task) + return SubprocessRuntime(env_file)(task) run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=placer) @@ -367,7 +367,7 @@ def placer(task: TaskRow) -> Any: async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None: - job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=LocalRuntime(env_file)) + job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(env_file)) (run,) = job.runs assert job.reward == 1.0 @@ -377,7 +377,7 @@ async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None: async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None: job = await _add_task(1, 2).run( - _FnAgent(_solve_add), runtime=LocalRuntime(env_file), group=2, max_concurrent=1 + _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file), group=2, max_concurrent=1 ) assert job.group == 2 @@ -388,7 +388,7 @@ async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None async def test_open_job_spans_multiple_scheduler_calls(env_file: Path) -> None: session = await Job.start("session", group=2) - provider = LocalRuntime(env_file) + provider = SubprocessRuntime(env_file) job1 = await _add_task(1, 1).run(_FnAgent(_solve_add), runtime=provider, job=session) job2 = await _add_task(2, 2).run(_FnAgent(_solve_add), runtime=provider, job=session) @@ -434,7 +434,7 @@ async def test_one_spawn_serves_each_rows_env_in_a_mixed_taskset( # One provider, two envs: each acquisition serves the row it was called # with (the task ids only exist on their own env, so a misplacement # would fail the rollout). - job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=LocalRuntime(path)) + job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(path)) assert [run.reward for run in job.runs] == [1.0, 1.0] assert [run.prompt for run in job.runs] == ["alpha:1:2", "beta:3:4"] @@ -444,7 +444,7 @@ async def test_rollout_threads_job_and_group_ids(env_file: Path) -> None: run = await rollout( _add_task(1, 1), _FnAgent(_solve_add), - runtime=LocalRuntime(env_file), + runtime=SubprocessRuntime(env_file), job_id="j1", group_id="g1", ) diff --git a/hud/tests/test_init.py b/hud/tests/test_init.py index b11dc88d2..144bed28f 100644 --- a/hud/tests/test_init.py +++ b/hud/tests/test_init.py @@ -55,6 +55,7 @@ def test_all_exports_available(self): "RuntimeLimits", "RuntimeResources", "LocalRuntime", + "SubprocessRuntime", "SyncPlan", "Task", "Taskset", diff --git a/hud/tests/test_init_module.py b/hud/tests/test_init_module.py index 45b458642..79a470974 100644 --- a/hud/tests/test_init_module.py +++ b/hud/tests/test_init_module.py @@ -35,6 +35,7 @@ def test_all_exports(self): "RuntimeLimits", "RuntimeResources", "LocalRuntime", + "SubprocessRuntime", "SyncPlan", "Task", "Taskset", From 24cc00bc2db8071c0077709f8d0c606f94a0e921 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:57:39 -0700 Subject: [PATCH 2/9] refactor(eval): LocalRuntime fresh form is build=(task) -> Environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mapping values are live Environments only — shared is the single semantic of the data form, no type-dispatch on values. The fresh case moves to an explicit build= callable that receives the placed row, which zero-arg factories could not express (per-row construction like one env per task dir). --- docs/v6/reference/runtime.mdx | 17 ++++--- hud/eval/runtime.py | 76 +++++++++++++++------------- hud/eval/tests/test_local_runtime.py | 38 ++++++++------ 3 files changed, 73 insertions(+), 58 deletions(-) diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index cc39495fa..e3efe5f81 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -76,16 +76,17 @@ The constructor for each built-in runtime: ### `LocalRuntime` ```python -LocalRuntime(envs) +LocalRuntime(envs) # shared: a live env, or {env_name: env} +LocalRuntime(build=fn) # fresh: fn(task) -> Environment per rollout ``` -- **`envs`** - a live `Environment`, or a mapping of env name to `Environment` / zero-arg factory - for mixed-env tasksets. Rows join by `task.env` name. - -An `Environment` *instance* is a shared substrate: its daemons start on the first acquisition and -stop after the last, every rollout shares the env's capabilities and state, and each rollout gets its -own control channel so concurrent runs never collide. A *factory* is fresh per acquisition - built, -served, and stopped around each rollout - for envs whose state a rollout mutates. +- **`envs`** - a live `Environment`, or a mapping of env name to `Environment` for mixed-env + tasksets. Rows join by `task.env` name. Shared substrate: daemons start on the first acquisition + and stop after the last, every rollout shares the env's capabilities and state, and each rollout + gets its own control channel so concurrent runs never collide. +- **`build`** - a callable receiving the placed task row and returning the env to serve for it. + Fresh per acquisition - built, served, and stopped around each rollout - for envs whose state a + rollout mutates, or whose construction depends on the row. Serving is in-process on a loopback port through the same control channel as every placement; only isolation differs. Env hooks share this process's event loop, so keep envs async - or use diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index 0015f9090..c2c2e28f9 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -7,8 +7,8 @@ elsewhere) are the same code, differing only in the url. - :class:`LocalRuntime` — serves live :class:`Environment` objects in-process - (rows join by env name; instances are shared substrates, factories fresh - per acquisition). + (rows join by env name; shared substrates, or ``build=`` for a fresh env + constructed from each placed row). - :class:`SubprocessRuntime` — runs a child process serving the row's env from a ``.py`` source (the path is always given, never recovered from a live object). @@ -161,20 +161,19 @@ def _modal_image_from_uri(modal: Any, image_uri: str) -> Any: class LocalRuntime: """The in-process provider: serve live :class:`Environment` objects from here. - The placement whose right-hand side is objects you already hold — a single - env, or a mapping of env name to env / zero-arg factory for a mixed-env - taskset. Rows join by ``task.env`` name, like every placement:: + Two forms, one per lifecycle. *Shared*: pass an env you already hold (or a + mapping of env name -> env for a mixed-env taskset); rows join by + ``task.env`` name, the env's daemons start on first acquisition and stop + after the last, and every rollout shares its capabilities and state — but + each gets its own control channel (a bound channel holds at most one + suspended task, so concurrent runs never collide). *Fresh*: pass + ``build=``, a callable receiving the placed row and returning the env to + serve for it — built, served, and stopped around each rollout, for envs + whose state a rollout mutates:: job = await taskset.run(agent, runtime=LocalRuntime(env)) - job = await taskset.run(agent, runtime=LocalRuntime({"tb-g1": make_g1})) - - An :class:`Environment` *instance* is a shared substrate: its daemons - start on first acquisition and stop after the last, and every rollout - placed on it shares the env's capabilities and state — but each gets its - own control channel (a bound channel holds at most one suspended task, so - concurrent runs never collide). A *factory* is fresh per acquisition — - built, served, and stopped around each rollout — for envs whose state a - rollout mutates. + job = await taskset.run(agent, runtime=LocalRuntime({"g1": g1, "g2": g2})) + job = await taskset.run(agent, runtime=LocalRuntime(build=env_for_row)) Serving is in-process on a loopback port, through the same control channel as any placement — only isolation differs: env hooks run in this process @@ -185,28 +184,39 @@ class LocalRuntime: def __init__( self, - envs: Environment | Mapping[str, Environment | Callable[[], Environment]], + envs: Environment | Mapping[str, Environment] | None = None, + *, + build: Callable[[Task], Environment] | None = None, ) -> None: if isinstance(envs, (str, Path)): raise TypeError( "LocalRuntime serves live Environment objects; " "use SubprocessRuntime(path) to serve a source file." ) + if (envs is None) == (build is None): + raise TypeError("LocalRuntime: pass exactly one of envs or build=") from hud.environment.env import Environment as _Environment - if isinstance(envs, Mapping): - self._envs: dict[str, _Environment | Callable[[], _Environment]] = dict(envs) + self._build = build + if envs is None: + self._envs: dict[str, _Environment] = {} elif isinstance(envs, _Environment): self._envs = {envs.name: envs} + elif isinstance(envs, Mapping): + bad = {name: e for name, e in envs.items() if not isinstance(e, _Environment)} + if bad: + raise TypeError( + f"LocalRuntime: mapping values must be live Environments, got {bad!r}; " + "for per-row construction pass build= instead" + ) + self._envs = dict(envs) + if not self._envs: + raise ValueError("LocalRuntime: no environments given") else: - # A factory has no name until called, so the join key must be - # explicit: pass factories as {env_name: factory}. raise TypeError( f"LocalRuntime: expected an Environment or a mapping of " - f"env name -> Environment/factory; got {envs!r}" + f"env name -> Environment; got {envs!r}" ) - if not self._envs: - raise ValueError("LocalRuntime: no environments given") # Shared-instance daemon refcounts, keyed by env name: start on first # acquisition, stop after the last. self._leases: dict[str, int] = {} @@ -218,21 +228,19 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: if task.runtime_config is not None: raise ValueError("LocalRuntime does not support task runtime_config") - entry = self._envs.get(task.env) - if entry is None: - raise KeyError( - f"LocalRuntime has no environment named {task.env!r} (has: {sorted(self._envs)})" - ) - if isinstance(entry, _Environment): - async with self._acquire_shared(task.env, entry) as runtime: + if self._build is not None: + env = self._build(task) + if not isinstance(env, _Environment): + raise TypeError(f"LocalRuntime build= returned {env!r}, not an Environment") + async with _local(env) as runtime: yield runtime return - env = entry() - if not isinstance(env, _Environment): - raise TypeError( - f"LocalRuntime factory for {task.env!r} returned {env!r}, not an Environment" + env = self._envs.get(task.env) + if env is None: + raise KeyError( + f"LocalRuntime has no environment named {task.env!r} (has: {sorted(self._envs)})" ) - async with _local(env) as runtime: + async with self._acquire_shared(task.env, env) as runtime: yield runtime @asynccontextmanager diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py index 9b676b417..e3aba7569 100644 --- a/hud/eval/tests/test_local_runtime.py +++ b/hud/eval/tests/test_local_runtime.py @@ -1,10 +1,10 @@ """LocalRuntime: the in-process placement over live ``Environment`` objects. -Rows join by env name like every placement; an ``Environment`` instance is a -shared substrate (daemons started once, refcounted across acquisitions, one -control channel per acquisition), a factory in a mapping is fresh per -acquisition. Everything still crosses the control channel — these tests drive -the real rollout engine against envs that only exist in this process. +Shared form: live envs (single or name-keyed mapping), daemons started once and +refcounted across acquisitions, one control channel per acquisition. Fresh +form: ``build=``, constructing an env from the placed row per acquisition. +Everything still crosses the control channel — these tests drive the real +rollout engine against envs that only exist in this process. """ from __future__ import annotations @@ -113,23 +113,22 @@ async def test_shared_env_runs_grouped_taskset() -> None: assert all(run.reward == 1.0 for run in job.runs) -async def test_factory_builds_fresh_env_per_acquisition() -> None: - built: list[Environment] = [] +async def test_build_makes_fresh_env_per_acquisition_from_the_row() -> None: + built: list[str] = [] - def factory() -> Environment: - env = _sums_env() - built.append(env) - return env + def build(task: Task) -> Environment: + built.append(task.env) + return _sums_env(task.env) task = Task(env="sums", id="add", args={"a": 1, "b": 2}) job = await task.run( _FnAgent(_solve_add), - runtime=LocalRuntime({"sums": factory}), + runtime=LocalRuntime(build=build), group=3, ) assert all(run.reward == 1.0 for run in job.runs) - assert len(built) == 3 + assert built == ["sums", "sums", "sums"] async def test_mapping_joins_rows_by_env_name() -> None: @@ -175,6 +174,13 @@ def test_rejects_path_argument_pointing_at_subprocess_runtime() -> None: LocalRuntime(cast("Any", "env.py")) -def test_rejects_bare_factory_without_a_name() -> None: - with pytest.raises(TypeError, match="mapping"): - LocalRuntime(cast("Any", _sums_env)) +def test_rejects_factory_as_mapping_value() -> None: + with pytest.raises(TypeError, match="build="): + LocalRuntime(cast("Any", {"sums": _sums_env})) + + +def test_requires_exactly_one_of_envs_or_build() -> None: + with pytest.raises(TypeError, match="exactly one"): + LocalRuntime() + with pytest.raises(TypeError, match="exactly one"): + LocalRuntime(_sums_env(), build=lambda task: _sums_env()) From 4006aabb3b71bf78371a04b8d585e5b3b90f2c4a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Tue, 7 Jul 2026 18:59:52 -0700 Subject: [PATCH 3/9] feat(eval): LocalRuntime takes any pointer to the env; no-runtime resolves from what is known LocalRuntime(source, env=, ready_timeout=) serves a fresh env per rollout in this process, from any pointer to it: a .py path (throwaway module import per acquisition), a live module-level Environment (its declaring file, located by identity in sys.modules, is the recipe - the instance itself is never served), or a (task) -> Environment constructor. ready_timeout bounds env.start(). runtime= stays Runtime-family typed; there is no shared local form - shared substrates are Runtime(url). SubprocessRuntime keeps the child-process form for isolation. Task rows stay pure data: _source is gone and rows never carry placement. With no runtime, what is already known decides: a module-loaded taskset serves its source directory; a platform taskset runs on the platform; rows naming envs declared in imported modules serve each fresh from its file (ambiguity across live envs is a loud error, resolution is logged); anything else raises, naming the forms to pass - there is no silent fallback to the platform. --- docs/v6/guides/running-an-eval.mdx | 7 +- docs/v6/guides/training-agents.mdx | 6 +- docs/v6/reference/agents.mdx | 4 +- docs/v6/reference/runtime.mdx | 54 +++--- docs/v6/reference/tasks.mdx | 4 +- docs/v6/reference/types.mdx | 2 +- docs/v6/start/overview.mdx | 9 +- hud/cli/templates.py | 6 +- hud/environment/__init__.py | 3 +- hud/environment/env.py | 2 +- hud/eval/__init__.py | 15 +- hud/eval/chat.py | 5 +- hud/eval/run.py | 2 +- hud/eval/runtime.py | 232 +++++++++++------------ hud/eval/task.py | 4 +- hud/eval/taskset.py | 81 +++++++- hud/eval/tests/test_local_runtime.py | 270 ++++++++++++++++----------- hud/eval/tests/test_task.py | 7 +- 18 files changed, 419 insertions(+), 294 deletions(-) diff --git a/docs/v6/guides/running-an-eval.mdx b/docs/v6/guides/running-an-eval.mdx index b83131bfd..c080693a7 100644 --- a/docs/v6/guides/running-an-eval.mdx +++ b/docs/v6/guides/running-an-eval.mdx @@ -112,14 +112,14 @@ is the same eval `hud eval` runs, written out in Python. ```python run.py import asyncio -from hud import Taskset, SubprocessRuntime +from hud import Taskset, LocalRuntime from hud.agents import create_agent agent = create_agent("claude-sonnet-4-5") ts = Taskset.from_file("tasks.py") async def main(): - job = await ts.run(agent, runtime=SubprocessRuntime("env.py")) + job = await ts.run(agent, runtime=LocalRuntime("env.py")) print(job.reward) asyncio.run(main()) @@ -130,8 +130,7 @@ change to `env.py` or the tasks: | Runtime | Where the env runs | | --- | --- | -| `LocalRuntime(env)` | This process, serving a live env object | -| `SubprocessRuntime("env.py")` | A child process on your machine | +| `LocalRuntime("env.py")` | In this process, on your machine | | `DockerRuntime("my-env")` | A fresh local container per rollout | | `ModalRuntime("my-env")` | A fresh [Modal](https://modal.com) sandbox per rollout | | `DaytonaRuntime("my-env")` | A fresh [Daytona](https://daytona.io) sandbox per rollout | diff --git a/docs/v6/guides/training-agents.mdx b/docs/v6/guides/training-agents.mdx index de055b4a7..11f1abf13 100644 --- a/docs/v6/guides/training-agents.mdx +++ b/docs/v6/guides/training-agents.mdx @@ -93,7 +93,7 @@ One job spans the session; each step appends a batch and trains on it: - **Open the job** with `group=8` - 8 rollouts per task, so the rewards are comparable (next). - **Roll out** the batch, the same eval as the [previous guide](/v6/guides/running-an-eval). The - [runtime](/v6/reference/runtime) sets where it runs; swap `SubprocessRuntime` for `HUDRuntime()` unchanged. + [runtime](/v6/reference/runtime) sets where it runs; swap `LocalRuntime` for `HUDRuntime()` unchanged. - **Nudge** with `trainer.step` - the one line that learns. It scores each rollout against its group, shifts the weights, then **promotes** them so the gateway serves the new ones at once. @@ -106,7 +106,7 @@ One job spans the session; each step appends a batch and trains on it: ```python import asyncio -from hud import TrainingClient, Taskset, SubprocessRuntime +from hud import TrainingClient, Taskset, LocalRuntime from hud.agents import create_agent from hud.eval import Job @@ -122,7 +122,7 @@ async def main(): session = await Job.start(MODEL, group=8) # one job spans the session for step in range(10): start = len(session.runs) - await taskset.run(agent, runtime=SubprocessRuntime("env.py"), job=session) + await taskset.run(agent, runtime=LocalRuntime("env.py"), job=session) batch = session.runs[start:] # this step's rollouts await trainer.step(batch, learning_rate=1e-5, group_size=8) # nudge + promote print(f"step {step} reward {sum(r.reward for r in batch) / len(batch):.2f}") diff --git a/docs/v6/reference/agents.mdx b/docs/v6/reference/agents.mdx index 3c91734ad..1f32b2961 100644 --- a/docs/v6/reference/agents.mdx +++ b/docs/v6/reference/agents.mdx @@ -105,11 +105,11 @@ with a [runtime](/v6/reference/runtime): ```python from hud.agents import create_agent -from hud.eval import SubprocessRuntime, Taskset +from hud.eval import LocalRuntime, Taskset agent = create_agent("claude-sonnet-4-5") taskset = Taskset.from_file("tasks.py") # scaffolded tasks.py exports a list of tasks -job = await taskset.run(agent, runtime=SubprocessRuntime("env.py")) +job = await taskset.run(agent, runtime=LocalRuntime("env.py")) print(job.reward) ``` diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index e3efe5f81..c894eaecf 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -9,18 +9,17 @@ A **runtime** chooses where each rollout's environment runs. You pass it to `tas runtime changes. ```python -from hud import LocalRuntime, SubprocessRuntime +from hud import LocalRuntime -await taskset.run(agent, runtime=LocalRuntime(env)) # serve a live env in-process -await taskset.run(agent, runtime=SubprocessRuntime("env.py")) # serve env.py in a child process +await taskset.run(agent, runtime=LocalRuntime("env.py")) # serve env.py locally, run here ``` ## Built-in runtimes | Runtime | Where the env runs | When to reach for it | |---------|--------------------|----------------------| -| `LocalRuntime(env)` | This process, serving live `Environment` objects | Fastest iteration; envs materialized in code | -| `SubprocessRuntime("env.py")` | A child process from your source | Local development with process isolation | +| `LocalRuntime("env.py")` | This process, loaded fresh from your source per rollout | Fastest iteration; local development | +| `SubprocessRuntime("env.py")` | A child process from your source | Local runs isolated from your process | | `DockerRuntime("my-env")` | A fresh local container per rollout | Reproducibility and parity with production | | `ModalRuntime("my-env")` | A fresh [Modal](https://modal.com/) sandbox per rollout | Cloud scale, no infra to manage | | `DaytonaRuntime("my-env")` | A fresh [Daytona](https://www.daytona.io/) sandbox per rollout | Cloud scale on Daytona | @@ -28,14 +27,17 @@ await taskset.run(agent, runtime=SubprocessRuntime("env.py")) # serve env.p | `HUDRuntime()` | A HUD-hosted env, leased by name and tunneled | Local agent loop against a deployed env | | `HostedRuntime()` | The whole rollout on a HUD-leased box | Agent and env run together off your machine | -Most runtimes are on the top-level package (`from hud import LocalRuntime, SubprocessRuntime, -DockerRuntime, HUDRuntime, HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from -`hud.eval`. +Most runtimes are on the top-level package (`from hud import LocalRuntime, DockerRuntime, HUDRuntime, +HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.eval`. -**Omit `runtime=`** and the run defaults to `HUDRuntime()` - the deployed env leased by each row's -`env` name. Tasks are pure data; a placement is always supplied at run time, never recovered from -the rows. +**Omit `runtime=`** and what is already known decides: a taskset loaded from local `.py` source +(`Taskset.from_file` / `from_module`) serves that source's directory; a platform taskset +(`from_api`) runs on the platform; tasks naming envs declared at module top level in modules you've +imported serve each fresh from its defining file — so `my_task().run(agent)` works in the same +project that defines the env. Anything else raises, naming the forms to pass — there is no silent +fallback. Tasks are pure data: rows never carry placement; resolution reads the taskset's origin and +the process at run time, and an ambiguous env name (declared by two live envs) is a loud error. To deploy an environment to the platform and run against it, see @@ -67,7 +69,7 @@ RuntimeConfig( Support differs per runtime: `DockerRuntime`, `ModalRuntime`, and `DaytonaRuntime` accept it (Docker ignores `limits`; Daytona ignores `run_timeout_s` and resource overrides when booting from a snapshot). -`LocalRuntime`, `SubprocessRuntime`, and `HUDRuntime` reject a per-task `runtime_config`. +`LocalRuntime` and `HUDRuntime` reject a per-task `runtime_config`. ## Runtime directory @@ -76,21 +78,23 @@ The constructor for each built-in runtime: ### `LocalRuntime` ```python -LocalRuntime(envs) # shared: a live env, or {env_name: env} -LocalRuntime(build=fn) # fresh: fn(task) -> Environment per rollout +LocalRuntime(source, *, env=None, ready_timeout=120.0) ``` -- **`envs`** - a live `Environment`, or a mapping of env name to `Environment` for mixed-env - tasksets. Rows join by `task.env` name. Shared substrate: daemons start on the first acquisition - and stop after the last, every rollout shares the env's capabilities and state, and each rollout - gets its own control channel so concurrent runs never collide. -- **`build`** - a callable receiving the placed task row and returning the env to serve for it. - Fresh per acquisition - built, served, and stopped around each rollout - for envs whose state a - rollout mutates, or whose construction depends on the row. - -Serving is in-process on a loopback port through the same control channel as every placement; only -isolation differs. Env hooks share this process's event loop, so keep envs async - or use -`SubprocessRuntime` / `DockerRuntime` when the env should not share the orchestrator's fate. +Serves a fresh env per rollout, in this process, over the same control channel as every placement. +`source` is any pointer to the env: + +- **a `.py` file or directory** that declares it, imported fresh per rollout (sibling imports + resolve). **`env`** pins one name when the source declares several; it defaults to the placed + task's env. +- **a live `Environment`** declared at module level - its declaring module's file is the recipe; + the instance itself is never served, so every rollout is still fresh. +- **a `(task) -> Environment` constructor** for envs built in code (integrations, parameterized + envs), called fresh per rollout with the placed row. + +`ready_timeout` bounds `@env.initialize` startup. Env hooks run in this process and share its event +loop - keep envs async, or use `SubprocessRuntime` / `DockerRuntime` when the env should not share +your process's fate. ### `SubprocessRuntime` diff --git a/docs/v6/reference/tasks.mdx b/docs/v6/reference/tasks.mdx index ef364f47a..0a764ed50 100644 --- a/docs/v6/reference/tasks.mdx +++ b/docs/v6/reference/tasks.mdx @@ -145,9 +145,9 @@ collections: `task.run(...)` is the same call over a taskset of one, with identical semantics. ```python -from hud import SubprocessRuntime +from hud import LocalRuntime -job = await ts.run(agent, runtime=SubprocessRuntime("env.py"), group=8, max_concurrent=10) +job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=8, max_concurrent=10) ``` | Parameter | Type | Description | diff --git a/docs/v6/reference/types.mdx b/docs/v6/reference/types.mdx index 600bc3b33..21c77c91c 100644 --- a/docs/v6/reference/types.mdx +++ b/docs/v6/reference/types.mdx @@ -67,7 +67,7 @@ run reports under a job, so even a single `task.run` returns a job of one. You g | `results` | `dict[str, list[Run]]` | Property: runs grouped by task slug - the alignment-safe alternative to `zip(tasks, runs)`, list-valued since `group > 1` gives several runs per task. | ```python -job = await ts.run(agent, runtime=SubprocessRuntime("env.py"), group=4) +job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=4) print(job.reward) # mean across every run ``` diff --git a/docs/v6/start/overview.mdx b/docs/v6/start/overview.mdx index f8ef66a10..3b5818278 100644 --- a/docs/v6/start/overview.mdx +++ b/docs/v6/start/overview.mdx @@ -155,10 +155,9 @@ hud eval env.py claude --runtime hud # same env, executed on HUD's hosted inf [runtime](/v6/reference/runtime) and run a taskset against it: ```python -from hud.eval import LocalRuntime, SubprocessRuntime, DockerRuntime, ModalRuntime, HUDRuntime +from hud.eval import LocalRuntime, DockerRuntime, ModalRuntime, HUDRuntime -LocalRuntime(env) # this process - live env objects -SubprocessRuntime("env.py") # local child process serving a source file +LocalRuntime("env.py") # in this process - fastest iteration DockerRuntime("my-env") # a fresh container per rollout ModalRuntime("my-env") # a Modal cloud sandbox per rollout HUDRuntime() # HUD's hosted infra (after `hud deploy`) @@ -200,12 +199,12 @@ You can run this programmatically: ```python from hud.agents import create_agent -from hud.eval import SubprocessRuntime +from hud.eval import LocalRuntime from tasks import TASKS agent = create_agent("claude-sonnet-4-5") # routed through the HUD gateway -job = await TASKS.run(agent, runtime=SubprocessRuntime("env.py")) # start the run +job = await TASKS.run(agent, runtime=LocalRuntime("env.py")) # start the run print(job.reward) ``` {/* diff --git a/hud/cli/templates.py b/hud/cli/templates.py index 28c560a5e..5be236857 100644 --- a/hud/cli/templates.py +++ b/hud/cli/templates.py @@ -87,10 +87,10 @@ async def test(): agent = ClaudeAgent() - # Calling a task binds a runnable Task; ``runtime=LocalRuntime(env)`` serves the - # live env in this process and runs the task against it over the wire. + # Calling a task binds a runnable Task; ``runtime=LocalRuntime(__file__)`` serves this + # file in a child process and runs the task against it over the wire. task = count(sentence="Strawberry world", letter="r") - job = await task.run(agent, runtime=LocalRuntime(env)) + job = await task.run(agent, runtime=LocalRuntime(__file__)) print("reward:", job.reward) diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index 0950965b6..94274f173 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -5,8 +5,7 @@ :mod:`~hud.environment.server` is the serving entry point substrates run. How a substrate comes up — placement — belongs to the eval engine: see :mod:`hud.eval.runtime` (:class:`~hud.eval.runtime.Runtime`, the ``Provider`` -contract, ``LocalRuntime``, ``SubprocessRuntime``, ``DockerRuntime``, -``HUDRuntime``). +contract, ``LocalRuntime``, ``DockerRuntime``, ``HUDRuntime``). The env-side robot runtime (bridges, action providers, sim runners, contract tooling, recording glue) lives in :mod:`hud.environment.robot`; import it diff --git a/hud/environment/env.py b/hud/environment/env.py index 960de529d..b3072e78c 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -76,7 +76,7 @@ class _TaskFactory(Generic[P]): binds a runnable :class:`~hud.eval.Task`:: task = fix_bug(difficulty=3) # -> Task - job = await task.run(agent, runtime=LocalRuntime(env)) + job = await task.run(agent, runtime=LocalRuntime("env.py")) """ def __init__( diff --git a/hud/eval/__init__.py b/hud/eval/__init__.py index 43765fc6c..2824645c7 100644 --- a/hud/eval/__init__.py +++ b/hud/eval/__init__.py @@ -13,17 +13,16 @@ exception: calling an ``@env.template`` declaration constructs the eval ``Task`` row.) -Placement is passed at execution time (see :mod:`.runtime`): ``LocalRuntime`` -live envs in this process, ``SubprocessRuntime`` a local source, -``DockerRuntime`` an image, ``Runtime(url)`` an env served elsewhere, -``HUDRuntime`` a HUD runtime tunnel, or ``HostedRuntime`` to run the whole -rollout remotely on the platform:: +Placement is passed at execution time (see :mod:`.runtime`): ``LocalRuntime`` a +local source served in-process, ``DockerRuntime`` an image, +``Runtime(url)`` an env served elsewhere, ``HUDRuntime`` a HUD runtime tunnel, +or ``HostedRuntime`` to run the whole rollout remotely on the platform:: - from hud.eval import LocalRuntime, SubprocessRuntime, Taskset + from hud.eval import LocalRuntime, Taskset - job = await my_task(a=1).run(agent, runtime=LocalRuntime(env)) + job = await my_task(a=1).run(agent, runtime=LocalRuntime("env.py")) job = await Taskset("demo", [my_task(d) for d in range(5)]).run( - agent, runtime=SubprocessRuntime("env.py"), group=8 + agent, runtime=LocalRuntime("env.py"), group=8 ) """ diff --git a/hud/eval/chat.py b/hud/eval/chat.py index 1e0e60730..836f41713 100644 --- a/hud/eval/chat.py +++ b/hud/eval/chat.py @@ -96,8 +96,7 @@ def __init__( (stateless per run, e.g. ``create_agent("claude-sonnet-4-5")``). runtime: The env placement each turn's rollout runs against — a :class:`~hud.eval.runtime.Provider` such as - ``LocalRuntime(env)``, ``SubprocessRuntime("env.py")``, or - ``Runtime("tcp://...")``. Chat is + ``LocalRuntime("env.py")`` or ``Runtime("tcp://...")``. Chat is interactive and local: it drives the agent loop in this process, so hosted placement does not apply. """ @@ -134,7 +133,7 @@ async def send(self, message: MessageContent) -> Trace: if self._runtime is None: raise RuntimeError( "Chat needs a runtime to converse against — pass an env placement, " - 'e.g. runtime=LocalRuntime(env) or runtime=Runtime("tcp://...").' + 'e.g. runtime=Runtime("tcp://...") or runtime=LocalRuntime("env.py").' ) if self.job is None: # one job spans the whole conversation self.job = await Job.start(self._task.id) diff --git a/hud/eval/run.py b/hud/eval/run.py index 2df207d06..5949786bd 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -5,7 +5,7 @@ loopback, a container, a cloud sandbox), starts the task, drives the agent, grades, and tears down, filling a :class:`Run` along the way:: - run = await rollout(task, agent, runtime=SubprocessRuntime("env.py")) + run = await rollout(task, agent, runtime=LocalRuntime("env.py")) It is the *client-here* path: the agent loop runs in this process against a :class:`~hud.eval.runtime.Provider`'s channel. The same driver runs on the diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index c2c2e28f9..ce7e40578 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -6,12 +6,12 @@ transparent, so "co-located" (loopback) and "split" (agent here, env elsewhere) are the same code, differing only in the url. -- :class:`LocalRuntime` — serves live :class:`Environment` objects in-process - (rows join by env name; shared substrates, or ``build=`` for a fresh env - constructed from each placed row). -- :class:`SubprocessRuntime` — runs a child process serving the row's env from - a ``.py`` source (the path is always given, never recovered from a live - object). +- :class:`LocalRuntime` — serve a fresh env per rollout, in this process, + from any pointer to it: a ``.py`` source path, a live module-level + :class:`Environment` (its declaring file is the recipe), or a + ``(task) -> Environment`` constructor. +- :class:`SubprocessRuntime` — serve the row's env from a ``.py`` source in a + child process, when the env should not share the orchestrator's fate. - :class:`DockerRuntime` — ``docker run``s an image whose CMD serves the channel. - ``Runtime(url)`` — the ``nullcontext`` of providers: yields itself, a *borrowed, shared* substrate provisioned elsewhere (env served anywhere — @@ -36,7 +36,6 @@ import sys import uuid from collections import deque -from collections.abc import Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from dataclasses import dataclass, field from pathlib import Path @@ -54,7 +53,7 @@ from .run import Grade, Run, rollout if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Sequence + from collections.abc import AsyncIterator, Callable, Mapping, Sequence from hud.agents.base import Agent from hud.environment.env import Environment @@ -159,68 +158,79 @@ def _modal_image_from_uri(modal: Any, image_uri: str) -> Any: class LocalRuntime: - """The in-process provider: serve live :class:`Environment` objects from here. - - Two forms, one per lifecycle. *Shared*: pass an env you already hold (or a - mapping of env name -> env for a mixed-env taskset); rows join by - ``task.env`` name, the env's daemons start on first acquisition and stop - after the last, and every rollout shares its capabilities and state — but - each gets its own control channel (a bound channel holds at most one - suspended task, so concurrent runs never collide). *Fresh*: pass - ``build=``, a callable receiving the placed row and returning the env to - serve for it — built, served, and stopped around each rollout, for envs - whose state a rollout mutates:: - - job = await taskset.run(agent, runtime=LocalRuntime(env)) - job = await taskset.run(agent, runtime=LocalRuntime({"g1": g1, "g2": g2})) - job = await taskset.run(agent, runtime=LocalRuntime(build=env_for_row)) - - Serving is in-process on a loopback port, through the same control channel - as any placement — only isolation differs: env hooks run in this process - and share its event loop, so blocking env code stalls concurrent rollouts. - Use :class:`SubprocessRuntime` or :class:`DockerRuntime` when the env - should not share the orchestrator's fate. + """The local provider: a fresh env per rollout, served in this process. + + *source* is any pointer to the env: a ``.py`` file or directory declaring + it (imported as a throwaway module per acquisition; sibling imports + resolve; *env* pins one name when it declares several), a live + :class:`~hud.environment.Environment` declared at module level (its + declaring module's file is the recipe — the instance itself is never + served, so every rollout is still fresh), or a + ``(task) -> Environment`` constructor for envs built in code:: + + runtime = LocalRuntime("env.py") + runtime = LocalRuntime(env) + runtime = LocalRuntime(harbor.environment_for) + + Each acquisition serves its fresh env on an ephemeral loopback port; + ``ready_timeout`` bounds ``@env.initialize`` startup. Env hooks run in + this process and share its event loop — blocking env code stalls + concurrent rollouts. Use :class:`SubprocessRuntime` or + :class:`DockerRuntime` for process isolation; ``Runtime(url)`` attaches + rollouts to a substrate served elsewhere. """ def __init__( self, - envs: Environment | Mapping[str, Environment] | None = None, + source: str | Path | Environment | Callable[[Task], Environment], *, - build: Callable[[Task], Environment] | None = None, + env: str | None = None, + ready_timeout: float = 120.0, ) -> None: - if isinstance(envs, (str, Path)): - raise TypeError( - "LocalRuntime serves live Environment objects; " - "use SubprocessRuntime(path) to serve a source file." - ) - if (envs is None) == (build is None): - raise TypeError("LocalRuntime: pass exactly one of envs or build=") from hud.environment.env import Environment as _Environment - self._build = build - if envs is None: - self._envs: dict[str, _Environment] = {} - elif isinstance(envs, _Environment): - self._envs = {envs.name: envs} - elif isinstance(envs, Mapping): - bad = {name: e for name, e in envs.items() if not isinstance(e, _Environment)} - if bad: + self.ready_timeout = ready_timeout + # A live instance may have been mutated since its module was imported; + # verify the fresh copy still declares its templates, so drift fails + # at acquisition with the cause named instead of "unknown task" later. + expected_templates: frozenset[str] = frozenset() + if isinstance(source, _Environment): + file = _declaring_file(source) + if file is None: raise TypeError( - f"LocalRuntime: mapping values must be live Environments, got {bad!r}; " - "for per-row construction pass build= instead" + f"LocalRuntime: env {source.name!r} is not declared at module " + "level in an importable file, so it cannot be rebuilt fresh per " + "rollout; pass its constructor instead: " + "LocalRuntime(lambda task: )" ) - self._envs = dict(envs) - if not self._envs: - raise ValueError("LocalRuntime: no environments given") + expected_templates = frozenset(source.tasks) + source, env = file, env or source.name + if isinstance(source, (str, Path)): + path, pinned = Path(source).resolve(), env + from hud.environment import load_environment + + def _load(task: Task) -> _Environment: + loaded = load_environment(path, name=pinned or task.env) + missing = expected_templates - loaded.tasks.keys() + if missing: + raise ValueError( + f"env {loaded.name!r} loaded from {path} lacks template(s) " + f"{sorted(missing)} present on the live instance — it was " + "modified after import; pass a constructor instead: " + "LocalRuntime(lambda task: )" + ) + return loaded + + self._build: Callable[[Task], _Environment] = _load + elif callable(source): + if env is not None: + raise TypeError("LocalRuntime: env= applies only to source paths") + self._build = source else: raise TypeError( - f"LocalRuntime: expected an Environment or a mapping of " - f"env name -> Environment; got {envs!r}" + f"LocalRuntime: expected a source path, a live Environment, or a " + f"(task) -> Environment constructor; got {source!r}" ) - # Shared-instance daemon refcounts, keyed by env name: start on first - # acquisition, stop after the last. - self._leases: dict[str, int] = {} - self._lock = asyncio.Lock() @asynccontextmanager async def __call__(self, task: Task) -> AsyncIterator[Runtime]: @@ -228,36 +238,37 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: if task.runtime_config is not None: raise ValueError("LocalRuntime does not support task runtime_config") - if self._build is not None: + try: env = self._build(task) - if not isinstance(env, _Environment): - raise TypeError(f"LocalRuntime build= returned {env!r}, not an Environment") - async with _local(env) as runtime: - yield runtime - return - env = self._envs.get(task.env) - if env is None: - raise KeyError( - f"LocalRuntime has no environment named {task.env!r} (has: {sorted(self._envs)})" - ) - async with self._acquire_shared(task.env, env) as runtime: + except RuntimeError as e: + # The source ran an event loop at import — usually an unguarded + # top-level run call; name the actual mistake. + if "running event loop" not in str(e): + raise + raise RuntimeError( + "the env source ran async code while being imported to place a " + 'rollout — guard top-level run calls with `if __name__ == "__main__":`' + ) from e + if not isinstance(env, _Environment): + raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") + async with _local(env, ready_timeout=self.ready_timeout) as runtime: yield runtime - @asynccontextmanager - async def _acquire_shared(self, name: str, env: Environment) -> AsyncIterator[Runtime]: - async with self._lock: - if self._leases.get(name, 0) == 0: - await env.start() - self._leases[name] = self._leases.get(name, 0) + 1 - try: - async with _bind_channel(env) as runtime: - yield runtime - finally: - async with self._lock: - self._leases[name] -= 1 - if self._leases[name] == 0: - del self._leases[name] - await env.stop() + +def _declaring_file(env: Environment) -> Path | None: + """The file of a loaded module holding *env* in its globals, else None. + + Located by identity, so re-importing the file re-declares this env; a + module without a file (a notebook ``__main__``) cannot. + """ + for module in list(sys.modules.values()): + module_file = getattr(module, "__file__", None) + module_vars = getattr(module, "__dict__", None) + if not module_file or not isinstance(module_vars, dict): + continue + if any(value is env for value in list(module_vars.values())): + return Path(module_file) + return None class SubprocessRuntime: @@ -710,42 +721,33 @@ async def _docker(*args: str, check: bool = True) -> tuple[str, str]: @asynccontextmanager -async def _bind_channel(env: Environment) -> AsyncIterator[Runtime]: - """Bind one control channel over an already-started env, as a runtime. +async def _local(env: Environment, *, ready_timeout: float | None = None) -> AsyncIterator[Runtime]: + """Substrate-side serving: a live env owned by *this* process, as a runtime. - Each bound channel holds at most one suspended task, so concurrent - rollouts on a shared env each get their own channel (see - ``LocalRuntime``); the env's daemon lifecycle is the caller's concern. + One env lifecycle (start → serve → stop) around one bound control + channel; ``ready_timeout`` bounds ``env.start()`` (initialize + hooks/daemons). ``LocalRuntime`` enters this per acquisition; code + already running *inside* a placed substrate adapts it (``AgentTool`` + sub-rollouts: ``runtime=lambda _: _local(env)``); test harnesses enter + it directly. """ from hud.environment.server import bind - server = await bind(env, "127.0.0.1", 0) - host, port = server.sockets[0].getsockname()[:2] - serve_task = asyncio.create_task(server.serve_forever()) + started = env.start() + await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started) try: - yield Runtime(f"tcp://{host}:{port}") - finally: - serve_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await serve_task - server.close() - with contextlib.suppress(Exception): - await server.wait_closed() - - -@asynccontextmanager -async def _local(env: Environment) -> AsyncIterator[Runtime]: - """Substrate-side serving: a live env owned by *this* process, as a runtime. - - One env lifecycle (start → serve → stop) around one bound channel — the - single-use form ``LocalRuntime`` builds on. Code already running *inside* - a placed substrate adapts it (``AgentTool`` sub-rollouts: - ``runtime=lambda _: _local(env)``); test harnesses enter it directly. - """ - await env.start() - try: - async with _bind_channel(env) as runtime: - yield runtime + server = await bind(env, "127.0.0.1", 0) + host, port = server.sockets[0].getsockname()[:2] + serve_task = asyncio.create_task(server.serve_forever()) + try: + yield Runtime(f"tcp://{host}:{port}") + finally: + serve_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await serve_task + server.close() + with contextlib.suppress(Exception): + await server.wait_closed() finally: await env.stop() diff --git a/hud/eval/task.py b/hud/eval/task.py index d942f462c..2f6eb2814 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -84,8 +84,8 @@ async def run( open ``job`` from :meth:`Job.start` to accumulate into), ``group`` repeats sharing a group_id, ``max_concurrent`` capping parallelism — over a taskset of one. ``runtime`` is the placement; left unset it - falls back to the HUD runtime tunnel by ``env`` name. To run against - a live env in this process, pass ``runtime=LocalRuntime(env)``. + falls back to the HUD runtime tunnel by ``env`` name. For a local + run, pass one explicitly (``runtime=LocalRuntime("env.py")``). """ from .taskset import Taskset # circular: taskset -> sync -> task diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index b9a8b296f..10071dda1 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -5,7 +5,7 @@ :mod:`hud.eval.job`; platform persistence in :mod:`hud.eval.sync`:: job = await Taskset("bugs", [fix_bug(difficulty=d) for d in range(5)]).run( - agent, runtime=SubprocessRuntime("env.py") + agent, runtime=LocalRuntime("env.py") ) """ @@ -14,16 +14,18 @@ import asyncio import json import logging +import sys import uuid from pathlib import Path from typing import TYPE_CHECKING, Any +from hud.environment.env import Environment from hud.telemetry import flush from hud.utils.platform import PlatformClient from .job import Job, job_enter from .run import rollout -from .runtime import HostedRuntime, HUDRuntime +from .runtime import HostedRuntime, HUDRuntime, LocalRuntime from .sync import fetch_taskset_tasks, resolve_taskset_id if TYPE_CHECKING: @@ -38,6 +40,38 @@ logger = logging.getLogger("hud.eval.taskset") +def _declared_env(name: str) -> Environment | None: + """A live env named *name* declared in a loaded module's globals, else None. + + The in-memory counterpart of :func:`~hud.environment.load_environment`'s + file scan: an env declared at module top level in an imported module can + be served fresh from its file (``LocalRuntime`` locates it). Distinct + envs claiming one name is ambiguous and raises; the same instance + re-exported across modules is one match. Envs not in any file-backed + module's globals (constructed inside a function, a notebook cell) + return None. + """ + matches: dict[int, tuple[Environment, str]] = {} + for module in list(sys.modules.values()): + module_file = getattr(module, "__file__", None) + module_vars = getattr(module, "__dict__", None) + if not module_file or not isinstance(module_vars, dict): + continue + for value in list(module_vars.values()): + if isinstance(value, Environment) and value.name == name: + matches.setdefault(id(value), (value, module_file)) + if not matches: + return None + if len(matches) > 1: + files = sorted({file for _, file in matches.values()}) + raise ValueError( + f"env name {name!r} is declared by multiple live environments " + f"({', '.join(files)}); pass runtime= explicitly — the exact " + "instance disambiguates: runtime=LocalRuntime(env)" + ) + return next(iter(matches.values()))[0] + + def _job_name(taskset_name: str, tasks: list[Task], group: int) -> str: suffix = f" ({group} times)" if group > 1 else "" if len(tasks) == 1: @@ -208,6 +242,30 @@ def environment_names(self) -> set[str]: """Return env names referenced by tasks in this taskset.""" return {task.env for task in self} + def _resolve_placement(self) -> Provider | HUDRuntime: + if self.origin and self.origin.startswith("module:"): + source = Path(self.origin[len("module:") :]) + return LocalRuntime(source if source.is_dir() else source.parent) + if self.origin and self.origin.startswith("api:"): + return HUDRuntime() + declared = {name: _declared_env(name) for name in self.environment_names()} + if declared and all(declared.values()): + providers = { + name: LocalRuntime(env) for name, env in declared.items() if env is not None + } + logger.info( + "no runtime given: serving %s fresh from their declaring modules", + ", ".join(sorted(providers)), + ) + return lambda task: providers[task.env](task) + missing = sorted(name for name, env in declared.items() if env is None) + raise ValueError( + f"no placement for env(s) {', '.join(missing) or ''}: pass runtime= — " + 'LocalRuntime("env.py") (a source file), LocalRuntime(env) (a live env), ' + "LocalRuntime(build) (a (task) -> Environment constructor), Runtime(url) " + "(a served substrate), or HUDRuntime() (your deployed env)" + ) + async def run( self, agent: Agent, @@ -224,7 +282,12 @@ async def run( placement: a :class:`~hud.eval.runtime.Provider` (the env served somewhere, the agent loop driven here by :func:`~hud.eval.run.rollout`), or :class:`~hud.eval.runtime.HostedRuntime` to run each rollout remotely - on the platform (left unset: HUD tunnel by env name). One provider serves a mixed-env + on the platform. Left unset, what is already known decides: a + taskset loaded from local ``.py`` source serves that source's + directory; a platform taskset runs on the platform; rows naming envs + declared in imported modules serve each fresh from its file; anything + else raises, naming the forms to pass. One provider serves a + mixed-env taskset and can size each substrate per row. Registers one HUD job as the platform receipt and reports each run's trace under it — or, given an open ``job`` (:meth:`Job.start`), accumulates this batch into it @@ -264,10 +327,14 @@ async def run( # Placement is chosen once for the batch: HostedRuntime delegates the # whole rollout to the platform, anything else is a Provider driven - # locally by rollout(). No runtime defaults to the HUD runtime tunnel - # by env name; live envs in this process are a placement too - # (``runtime=LocalRuntime(env)``), never recovered from the rows. - placement = runtime if runtime is not None else HUDRuntime() + # locally by rollout(). No runtime: what the taskset or this process + # already knows decides (rows never carry placement) — a loaded + # taskset runs where it came from; rows naming envs declared in + # imported modules serve each fresh from its file; anything else is + # an error naming the forms to pass. + if runtime is None: + runtime = self._resolve_placement() + placement = runtime sem = asyncio.Semaphore(max_concurrent) if max_concurrent else None async def _run(task: Task, group_id: str) -> Run: diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py index e3aba7569..c4c4b1db7 100644 --- a/hud/eval/tests/test_local_runtime.py +++ b/hud/eval/tests/test_local_runtime.py @@ -1,15 +1,17 @@ -"""LocalRuntime: the in-process placement over live ``Environment`` objects. - -Shared form: live envs (single or name-keyed mapping), daemons started once and -refcounted across acquisitions, one control channel per acquisition. Fresh -form: ``build=``, constructing an env from the placed row per acquisition. -Everything still crosses the control channel — these tests drive the real -rollout engine against envs that only exist in this process. +"""Local placement: LocalRuntime and the no-runtime resolution ladder. + +LocalRuntime serves a fresh env per rollout from any pointer to it — a source +path (throwaway import), a live module-level env (its declaring file is the +recipe), or a ``(task) -> Environment`` constructor. With no runtime, a run +uses what is already known: taskset origin, then envs declared in imported +modules, else a loud error. Everything crosses the real control channel — +these tests drive the rollout engine end to end. """ from __future__ import annotations -import asyncio +import importlib.util +import sys from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime from typing import Any, cast @@ -20,6 +22,37 @@ from hud.eval import LocalRuntime, Task, Taskset from hud.eval.run import rollout +_SUMS_ENV = """\ +from hud import Environment + +env = Environment("{name}") + + +@env.template(id="add") +async def add(a: int, b: int): + answer = yield f"add:{{a}}:{{b}}" + yield 1.0 if answer == str(a + b) else 0.0 +""" + + +@pytest.fixture +def imported_env(tmp_path, request): + """Write an env module, import it for real, and clean it up after. + + The module stays in ``sys.modules`` for the test's duration — the state a + user's ``from env import add`` leaves behind. + """ + module_name = f"_sums_mod_{request.node.name}" + file = tmp_path / f"{module_name}.py" + file.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") + spec = importlib.util.spec_from_file_location(module_name, file) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + yield module + del sys.modules[module_name] + def _sums_env(name: str = "sums") -> Environment: env = Environment(name) @@ -47,140 +80,163 @@ def _solve_add(prompt: str) -> str: return str(int(a) + int(b)) -async def test_live_env_rollout_end_to_end() -> None: +# ─── LocalRuntime: the three pointer forms ───────────────────────────── + + +async def test_source_path_serves_a_fresh_env_per_rollout(tmp_path) -> None: + # LOADS is module state: a fresh throwaway import per acquisition means + # every rollout sees exactly one load. + (tmp_path / "env.py").write_text( + "from hud import Environment\n\n" + "LOADS = []\n" + 'env = Environment("sums")\n\n\n' + '@env.template(id="add")\nasync def add(a: int, b: int):\n' + " LOADS.append(1)\n" + ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n' + " yield 1.0 if answer == str(a + b) else 0.0\n", + encoding="utf-8", + ) + + def _solve(prompt: str) -> str: + _, a, b, loads = prompt.split(":") + assert loads == "1" + return str(int(a) + int(b)) + + job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( + _FnAgent(_solve), + runtime=LocalRuntime(tmp_path / "env.py"), + group=2, + ) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + + +async def test_live_env_pointer_resolves_to_its_declaring_file(imported_env) -> None: run = await rollout( Task(env="sums", id="add", args={"a": 2, "b": 3}), _FnAgent(_solve_add), - runtime=LocalRuntime(_sums_env()), + runtime=LocalRuntime(imported_env.env), ) assert run.reward == 1.0 - assert run.trace_id - - -async def test_shared_instance_serves_once_across_concurrent_acquisitions() -> None: - env = _sums_env() - starts, stops = [], [] - - @env.initialize - async def _up() -> None: - starts.append(1) - - @env.shutdown - async def _down() -> None: - stops.append(1) - - provider = LocalRuntime(env) - task = Task(env="sums", id="add") - release = asyncio.Event() - all_acquired = asyncio.Event() - urls: list[str] = [] - - async def _hold() -> None: - async with provider(task) as runtime: - urls.append(runtime.url) - if len(urls) == 3: - all_acquired.set() - await release.wait() - - holders = [asyncio.create_task(_hold()) for _ in range(3)] - await all_acquired.wait() - # One shared env (daemons started once), but one channel per acquisition - # so concurrent task lifecycles never collide. - assert len(set(urls)) == 3 - assert starts == [1] - assert stops == [] - - release.set() - await asyncio.gather(*holders) - assert stops == [1] - - -async def test_shared_env_runs_grouped_taskset() -> None: - taskset = Taskset( - "sums", - [Task(env="sums", id="add", args={"a": a, "b": a + 1}, slug=f"add-{a}") for a in range(3)], - ) - job = await taskset.run( - _FnAgent(_solve_add), - runtime=LocalRuntime(_sums_env()), - group=2, - max_concurrent=3, - ) - assert len(job.runs) == 6 - assert all(run.reward == 1.0 for run in job.runs) +def test_live_env_without_a_declaring_file_is_rejected() -> None: + with pytest.raises(TypeError, match="constructor instead"): + LocalRuntime(_sums_env()) -async def test_build_makes_fresh_env_per_acquisition_from_the_row() -> None: +async def test_constructor_builds_fresh_per_rollout_from_the_row() -> None: built: list[str] = [] - def build(task: Task) -> Environment: + def env_for(task: Task) -> Environment: built.append(task.env) return _sums_env(task.env) - task = Task(env="sums", id="add", args={"a": 1, "b": 2}) - job = await task.run( + job = await Task(env="sums", id="add", args={"a": 1, "b": 2}).run( _FnAgent(_solve_add), - runtime=LocalRuntime(build=build), + runtime=LocalRuntime(env_for), group=3, + max_concurrent=3, ) assert all(run.reward == 1.0 for run in job.runs) assert built == ["sums", "sums", "sums"] -async def test_mapping_joins_rows_by_env_name() -> None: - doubles = Environment("doubles") +async def test_source_missing_env_name_fails_loudly(tmp_path) -> None: + (tmp_path / "env.py").write_text( + 'from hud import Environment\n\nenv = Environment("sums")\n', + encoding="utf-8", + ) + provider = LocalRuntime(tmp_path / "env.py") - @doubles.template(id="double") - async def double(n: int) -> AsyncGenerator[Any, Any]: - answer = yield f"double:{n}" - yield 1.0 if answer == str(2 * n) else 0.0 + with pytest.raises(ValueError, match="no Environment named 'other'"): + async with provider(Task(env="other", id="add")): + pass - def _solve(prompt: str) -> str: - kind, *parts = prompt.split(":") - if kind == "double": - return str(2 * int(parts[0])) - return str(int(parts[0]) + int(parts[1])) - - taskset = Taskset( - "mixed", - [ - Task(env="sums", id="add", args={"a": 4, "b": 5}), - Task(env="doubles", id="double", args={"n": 7}), - ], + +# ─── the no-runtime resolution ladder ────────────────────────────────── + + +async def test_module_loaded_taskset_serves_its_source_by_default(tmp_path, request) -> None: + (tmp_path / "env.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") + (tmp_path / "tasks.py").write_text( + "from env import add\n\ntasks = [add(a=2, b=3), add(a=4, b=5)]\n", + encoding="utf-8", ) + # tasks.py's `from env import add` imports env.py normally, so it outlives + # the throwaway tasks module. + request.addfinalizer(lambda: sys.modules.pop("env", None)) - job = await taskset.run( - _FnAgent(_solve), - runtime=LocalRuntime({"sums": _sums_env(), "doubles": doubles}), + taskset = Taskset.from_module(tmp_path / "tasks.py") + job = await taskset.run(_FnAgent(_solve_add)) + + assert len(job.runs) == 2 + assert all(run.reward == 1.0 for run in job.runs) + + +async def test_minted_tasks_resolve_a_declared_env_by_name(imported_env) -> None: + job = await Taskset("sums", [imported_env.add(a=2, b=3), imported_env.add(a=4, b=5)]).run( + _FnAgent(_solve_add) ) - assert [run.reward for run in job.runs] == [1.0, 1.0] + assert len(job.runs) == 2 + assert all(run.reward == 1.0 for run in job.runs) -async def test_unknown_env_name_fails_loudly() -> None: - provider = LocalRuntime(_sums_env()) +async def test_no_placement_fails_with_the_forms_to_pass() -> None: + with pytest.raises(ValueError, match="no placement for env"): + await Task(env="ghost", id="add").run(_FnAgent(_solve_add)) - with pytest.raises(KeyError, match="no environment named 'other'"): - async with provider(Task(env="other", id="add")): - pass +async def test_ambiguous_env_name_fails_loudly(imported_env, tmp_path, request) -> None: + module_name = f"_sums_rival_{request.node.name}" + file = tmp_path / f"{module_name}.py" + file.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") + spec = importlib.util.spec_from_file_location(module_name, file) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + with pytest.raises(ValueError, match="LocalRuntime\\(env\\)"): + await Task(env="sums", id="add").run(_FnAgent(_solve_add)) + finally: + del sys.modules[module_name] -def test_rejects_path_argument_pointing_at_subprocess_runtime() -> None: - with pytest.raises(TypeError, match="SubprocessRuntime"): - LocalRuntime(cast("Any", "env.py")) +def test_rejects_a_non_pointer_argument() -> None: + with pytest.raises(TypeError, match="expected a source path"): + LocalRuntime(cast("Any", 42)) -def test_rejects_factory_as_mapping_value() -> None: - with pytest.raises(TypeError, match="build="): - LocalRuntime(cast("Any", {"sums": _sums_env})) +# ─── seam defenses ───────────────────────────────────────────────────── + + +@pytest.mark.filterwarnings("ignore::RuntimeWarning") # the broken source leaks its coroutine +async def test_unguarded_run_call_in_source_names_the_mistake(tmp_path) -> None: + (tmp_path / "env.py").write_text( + "import asyncio\n\nfrom hud import Environment\n\n" + 'env = Environment("sums")\n\n' + "asyncio.run(asyncio.sleep(0))\n", + encoding="utf-8", + ) + provider = LocalRuntime(tmp_path / "env.py") -def test_requires_exactly_one_of_envs_or_build() -> None: - with pytest.raises(TypeError, match="exactly one"): - LocalRuntime() - with pytest.raises(TypeError, match="exactly one"): - LocalRuntime(_sums_env(), build=lambda task: _sums_env()) + with pytest.raises(RuntimeError, match='if __name__ == "__main__"'): + async with provider(Task(env="sums", id="add")): + pass + + +async def test_live_env_mutated_after_import_fails_with_the_cause(imported_env) -> None: + @imported_env.env.template(id="patched") + async def patched() -> Any: + answer = yield "noop" + yield 1.0 if answer else 0.0 + + provider = LocalRuntime(imported_env.env) + + with pytest.raises(ValueError, match="modified after import"): + async with provider(Task(env="sums", id="patched")): + pass diff --git a/hud/eval/tests/test_task.py b/hud/eval/tests/test_task.py index cf258a0d7..5d0de0586 100644 --- a/hud/eval/tests/test_task.py +++ b/hud/eval/tests/test_task.py @@ -137,7 +137,7 @@ def test_row_validation_rejects_malformed_entries() -> None: # ─── placement ───────────────────────────────────────────────────────── -async def test_no_placement_defaults_to_hud_runtime(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_platform_taskset_defaults_to_hud_runtime(monkeypatch: pytest.MonkeyPatch) -> None: import hud.eval.taskset as taskset_mod seen: dict[str, object] = {} @@ -150,8 +150,9 @@ async def fake_rollout(task: Task, agent: Agent, **kwargs: object) -> Run: monkeypatch.setattr(taskset_mod, "rollout", fake_rollout) - v = Task(env="hosted-env", id="solve", args={"n": 1}) - job = await v.run(cast("Agent", object())) + task = Task(env="hosted-env", id="solve", args={"n": 1}) + taskset = taskset_mod.Taskset("hosted", [task], origin="api:ts_123") + job = await taskset.run(cast("Agent", object())) (run,) = job.runs assert run.trace.status == "completed" From 5e46233bcb91b93b339083865919917b3b6e7c80 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:19:15 -0700 Subject: [PATCH 4/9] docs(eval): truth-up _local docstring, Chat placement error, CLI --runtime local docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _local's docstring claimed AgentTool sub-rollouts enter it; nothing in hud/agents does — its consumers are LocalRuntime (per acquisition, with the fresh env it built) and test harnesses. Chat's missing-runtime error now names the same placement forms as the taskset resolution error. The CLI --runtime local row says what the mechanism is: a child process per rollout (SubprocessRuntime). --- docs/v6/reference/cli.mdx | 2 +- hud/eval/chat.py | 5 +++-- hud/eval/runtime.py | 6 ++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index 6a5f51bbd..ca853a8a7 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -105,7 +105,7 @@ For a platform taskset, pass its name or id directly: `hud eval "My Tasks" claud | `--config`, `-c` | Agent config `key=value` (repeatable). | | `--verbose`, `-v` | Show agent logs (step progress, tool calls) for batch runs too. | | `--very-verbose`, `-vv` | Debug-level logs. | -| `--runtime` | Placement: `local`, `hud` (HUD runtime tunnel), or `tcp://host:port`. Defaults to `local` for a tasks file; platform tasksets default to remote hosted execution. | +| `--runtime` | Placement: `local` (a child process per rollout, serving the env source — `SubprocessRuntime`), `hud` (HUD runtime tunnel), or `tcp://host:port`. Defaults to `local` for a tasks file; platform tasksets default to remote hosted execution. | | `--remote` | Run the whole rollout remotely on the HUD platform. | | `--yes`, `-y` | Skip confirmation prompt. | diff --git a/hud/eval/chat.py b/hud/eval/chat.py index 836f41713..17fee982d 100644 --- a/hud/eval/chat.py +++ b/hud/eval/chat.py @@ -132,8 +132,9 @@ async def send(self, message: MessageContent) -> Trace: ) if self._runtime is None: raise RuntimeError( - "Chat needs a runtime to converse against — pass an env placement, " - 'e.g. runtime=Runtime("tcp://...") or runtime=LocalRuntime("env.py").' + "Chat needs a runtime to converse against — pass an env placement: " + 'LocalRuntime("env.py") (a source file), LocalRuntime(env) (a live env), ' + "Runtime(url) (a served substrate), or HUDRuntime() (your deployed env)." ) if self.job is None: # one job spans the whole conversation self.job = await Job.start(self._task.id) diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index ce7e40578..2d918ea43 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -726,10 +726,8 @@ async def _local(env: Environment, *, ready_timeout: float | None = None) -> Asy One env lifecycle (start → serve → stop) around one bound control channel; ``ready_timeout`` bounds ``env.start()`` (initialize - hooks/daemons). ``LocalRuntime`` enters this per acquisition; code - already running *inside* a placed substrate adapts it (``AgentTool`` - sub-rollouts: ``runtime=lambda _: _local(env)``); test harnesses enter - it directly. + hooks/daemons). ``LocalRuntime`` enters this per acquisition with the + fresh env it built; test harnesses enter it directly with a live one. """ from hud.environment.server import bind From c4f84654291f29510f07b9abe8ca8f7d56085468 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:32:29 -0700 Subject: [PATCH 5/9] fix(eval): origin claims rows only when it declares their envs; stop runs on failed startup Addresses the three real bot findings on #484: - The module: origin rung probed nothing and broadened a file to its parent directory. Both halves bit: a tasks-only module importing its envs from elsewhere failed instead of falling through to the live-env resolution, and a single-file taskset could hit 'multiple Environments' from a same-named sibling variant. The rung now claims the rows only when the origin source actually declares their env names, and serves the exact origin path. - _local awaited env.start() outside its try/finally, so a failed or timed-out initialize hook leaked already-started daemons; stop() now runs on startup failure (shutdown hooks are best-effort per hook). - The in-process freshness boundary is documented where it lives: the env's own source is re-imported per rollout, while modules it imports follow normal import caching and are shared process-wide - SubprocessRuntime is the whole-process isolation option. (Purging sys.modules per rollout would trade a documented boundary for reload hazards.) --- docs/v6/reference/runtime.mdx | 8 ++-- hud/eval/runtime.py | 20 +++++---- hud/eval/taskset.py | 19 ++++++++- hud/eval/tests/test_local_runtime.py | 61 ++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 11 deletions(-) diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index c894eaecf..73fb14096 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -92,9 +92,11 @@ Serves a fresh env per rollout, in this process, over the same control channel a - **a `(task) -> Environment` constructor** for envs built in code (integrations, parameterized envs), called fresh per rollout with the placed row. -`ready_timeout` bounds `@env.initialize` startup. Env hooks run in this process and share its event -loop - keep envs async, or use `SubprocessRuntime` / `DockerRuntime` when the env should not share -your process's fate. +`ready_timeout` bounds `@env.initialize` startup. The freshness boundary is the env's own source: +it is re-imported per rollout, while modules it imports follow normal Python import caching and are +shared process-wide - state kept in helper modules persists across rollouts. Env hooks run in this +process and share its event loop - keep envs async, or use `SubprocessRuntime` / `DockerRuntime` +when rollouts need whole-process isolation. ### `SubprocessRuntime` diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index 2d918ea43..a661d3c14 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -173,11 +173,15 @@ class LocalRuntime: runtime = LocalRuntime(harbor.environment_for) Each acquisition serves its fresh env on an ephemeral loopback port; - ``ready_timeout`` bounds ``@env.initialize`` startup. Env hooks run in - this process and share its event loop — blocking env code stalls - concurrent rollouts. Use :class:`SubprocessRuntime` or - :class:`DockerRuntime` for process isolation; ``Runtime(url)`` attaches - rollouts to a substrate served elsewhere. + ``ready_timeout`` bounds ``@env.initialize`` startup. The freshness + boundary is the env's own source: it is re-imported per rollout, while + modules it imports follow normal Python import caching and are shared + process-wide — state an env keeps in helper modules persists across + rollouts. Env hooks also run in this process and share its event loop, + so blocking env code stalls concurrent rollouts. Use + :class:`SubprocessRuntime` or :class:`DockerRuntime` when rollouts need + whole-process isolation; ``Runtime(url)`` attaches rollouts to a + substrate served elsewhere. """ def __init__( @@ -731,9 +735,11 @@ async def _local(env: Environment, *, ready_timeout: float | None = None) -> Asy """ from hud.environment.server import bind - started = env.start() - await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started) + # start() inside the try: a failed or timed-out initialize hook still gets + # its already-started daemons torn down by stop() (best-effort per hook). try: + started = env.start() + await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started) server = await bind(env, "127.0.0.1", 0) host, port = server.sockets[0].getsockname()[:2] serve_task = asyncio.create_task(server.serve_forever()) diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index 10071dda1..9da2e59e2 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -72,6 +72,18 @@ def _declared_env(name: str) -> Environment | None: return next(iter(matches.values()))[0] +def _declared_names(source: Path) -> set[str]: + """Env names a ``.py`` source (file or directory) declares at module level.""" + from hud.utils.modules import iter_modules + + return { + value.name + for module in iter_modules(source) + for value in vars(module).values() + if isinstance(value, Environment) + } + + def _job_name(taskset_name: str, tasks: list[Task], group: int) -> str: suffix = f" ({group} times)" if group > 1 else "" if len(tasks) == 1: @@ -244,8 +256,13 @@ def environment_names(self) -> set[str]: def _resolve_placement(self) -> Provider | HUDRuntime: if self.origin and self.origin.startswith("module:"): + # The origin claims the rows only if it actually declares their + # envs (a tasks-only module importing its envs from elsewhere + # does not) — and it serves as the exact path, so a same-named + # variant in a sibling file is never dragged in. source = Path(self.origin[len("module:") :]) - return LocalRuntime(source if source.is_dir() else source.parent) + if self.environment_names() <= _declared_names(source): + return LocalRuntime(source) if self.origin and self.origin.startswith("api:"): return HUDRuntime() declared = {name: _declared_env(name) for name in self.environment_names()} diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py index c4c4b1db7..5e33ab7eb 100644 --- a/hud/eval/tests/test_local_runtime.py +++ b/hud/eval/tests/test_local_runtime.py @@ -211,6 +211,67 @@ def test_rejects_a_non_pointer_argument() -> None: LocalRuntime(cast("Any", 42)) +async def test_failed_startup_still_runs_shutdown_hooks() -> None: + from hud.eval.runtime import _local + + env = _sums_env() + lifecycle: list[str] = [] + + @env.initialize + async def _up() -> None: + lifecycle.append("up") + + @env.shutdown + async def _down() -> None: + lifecycle.append("down") + + @env.initialize + async def _boom() -> None: + raise RuntimeError("daemon failed to start") + + with pytest.raises(RuntimeError, match="daemon failed to start"): + async with _local(env): + pass + + assert lifecycle == ["up", "down"] + + +async def test_tasks_only_module_resolves_envs_imported_from_elsewhere( + tmp_path, monkeypatch, request +) -> None: + # The env lives in a separate importable package, not next to tasks.py: + # the origin declares no envs, so resolution falls through to the live + # env the tasks module imported. + env_dir = tmp_path / "pkg" + env_dir.mkdir() + (env_dir / "sums_envmod.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") + tasks_dir = tmp_path / "tasks" + tasks_dir.mkdir() + (tasks_dir / "tasks.py").write_text( + "from sums_envmod import add\n\ntasks = [add(a=2, b=3)]\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(env_dir)) + request.addfinalizer(lambda: sys.modules.pop("sums_envmod", None)) + + taskset = Taskset.from_module(tasks_dir / "tasks.py") + job = await taskset.run(_FnAgent(_solve_add)) + + assert [run.reward for run in job.runs] == [1.0] + + +async def test_single_file_taskset_never_drags_in_a_same_named_sibling(tmp_path) -> None: + (tmp_path / "env_a.py").write_text( + _SUMS_ENV.format(name="sums") + "\ntasks = [add(a=2, b=3)]\n", encoding="utf-8" + ) + (tmp_path / "env_b.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") + + taskset = Taskset.from_module(tmp_path / "env_a.py") + job = await taskset.run(_FnAgent(_solve_add)) + + assert [run.reward for run in job.runs] == [1.0] + + # ─── seam defenses ───────────────────────────────────────────────────── From a1323cdc79f30957a0fb301ae0d8709768a8640e Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:40:30 -0700 Subject: [PATCH 6/9] docs(eval): rewrite the omit-runtime note for users --- docs/v6/reference/runtime.mdx | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index 73fb14096..102760d83 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -31,13 +31,17 @@ Most runtimes are on the top-level package (`from hud import LocalRuntime, Docke HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.eval`. -**Omit `runtime=`** and what is already known decides: a taskset loaded from local `.py` source -(`Taskset.from_file` / `from_module`) serves that source's directory; a platform taskset -(`from_api`) runs on the platform; tasks naming envs declared at module top level in modules you've -imported serve each fresh from its defining file — so `my_task().run(agent)` works in the same -project that defines the env. Anything else raises, naming the forms to pass — there is no silent -fallback. Tasks are pure data: rows never carry placement; resolution reads the taskset's origin and -the process at run time, and an ambiguous env name (declared by two live envs) is a loud error. +**You can usually omit `runtime=`.** A run without one uses what HUD already knows: + +- a taskset loaded from Python source (`Taskset.from_file` / `from_module`) runs against that source +- a platform taskset (`Taskset.from_api`) runs on the platform +- otherwise, if the envs your tasks name are defined in files you've imported, each rollout gets a + fresh env served from its file — so `my_task().run(agent)` just works in the project that defines + the env + +When none of these apply, `run` raises and lists the runtimes you can pass — it never silently picks +one. If two imported files define an env with the same name, that's also an error; disambiguate by +passing the instance you mean: `runtime=LocalRuntime(env)`. To deploy an environment to the platform and run against it, see From bb4453174958d3ba411843f7b101d4de6d0af728 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:46:49 -0700 Subject: [PATCH 7/9] refactor(eval): one home for env discovery; plain LocalRuntime docstring The three discovery helpers had sprawled across two modules with two separate sys.modules scans; they now share one iterator (_live_envs) next to their consumers in runtime.py, and taskset.py imports them instead of defining its own. LocalRuntime's docstring lists the three source forms as bullets and drops a reference to a function that does not exist. --- hud/eval/runtime.py | 96 ++++++++++++++++++++++++++++++++------------- hud/eval/taskset.py | 48 +---------------------- 2 files changed, 69 insertions(+), 75 deletions(-) diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index a661d3c14..e9a589ab3 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -53,7 +53,7 @@ from .run import Grade, Run, rollout if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Mapping, Sequence + from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence from hud.agents.base import Agent from hud.environment.env import Environment @@ -158,30 +158,30 @@ def _modal_image_from_uri(modal: Any, image_uri: str) -> Any: class LocalRuntime: - """The local provider: a fresh env per rollout, served in this process. + """The local provider: serve a fresh env per rollout, in this process. - *source* is any pointer to the env: a ``.py`` file or directory declaring - it (imported as a throwaway module per acquisition; sibling imports - resolve; *env* pins one name when it declares several), a live - :class:`~hud.environment.Environment` declared at module level (its - declaring module's file is the recipe — the instance itself is never - served, so every rollout is still fresh), or a - ``(task) -> Environment`` constructor for envs built in code:: + *source* points at the env in whatever form you have: + + - a ``.py`` file or directory — imported fresh per acquisition (sibling + imports resolve); *env* pins one name when several are declared, + defaulting to the placed task's env + - a live :class:`~hud.environment.Environment` — shorthand for its + declaring file; the instance itself is never served + - a ``(task) -> Environment`` callable — called per acquisition with the + placed row + + :: runtime = LocalRuntime("env.py") runtime = LocalRuntime(env) - runtime = LocalRuntime(harbor.environment_for) - - Each acquisition serves its fresh env on an ephemeral loopback port; - ``ready_timeout`` bounds ``@env.initialize`` startup. The freshness - boundary is the env's own source: it is re-imported per rollout, while - modules it imports follow normal Python import caching and are shared - process-wide — state an env keeps in helper modules persists across - rollouts. Env hooks also run in this process and share its event loop, - so blocking env code stalls concurrent rollouts. Use - :class:`SubprocessRuntime` or :class:`DockerRuntime` when rollouts need - whole-process isolation; ``Runtime(url)`` attaches rollouts to a - substrate served elsewhere. + runtime = LocalRuntime(lambda task: build_env(task.env)) + + ``ready_timeout`` bounds ``@env.initialize`` startup. Freshness covers + the env's own source; modules it imports are cached as usual and shared + across rollouts. Hooks share this process's event loop, so blocking env + code stalls concurrent rollouts — use :class:`SubprocessRuntime` or + :class:`DockerRuntime` for process isolation, and ``Runtime(url)`` to + attach to a substrate served elsewhere. """ def __init__( @@ -259,20 +259,60 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: yield runtime -def _declaring_file(env: Environment) -> Path | None: - """The file of a loaded module holding *env* in its globals, else None. +def _live_envs() -> Iterator[tuple[Environment, str]]: + """Envs declared in loaded, file-backed modules' globals, with their files. - Located by identity, so re-importing the file re-declares this env; a - module without a file (a notebook ``__main__``) cannot. + The in-memory counterpart of scanning ``.py`` sources on disk + (:func:`~hud.environment.load_environment`): an env found here can be + served fresh by re-importing its file. Envs in modules without a file + (a notebook ``__main__``) are not yielded — re-import could not + reconstruct them. """ + from hud.environment.env import Environment as _Environment + for module in list(sys.modules.values()): module_file = getattr(module, "__file__", None) module_vars = getattr(module, "__dict__", None) if not module_file or not isinstance(module_vars, dict): continue - if any(value is env for value in list(module_vars.values())): - return Path(module_file) - return None + for value in list(module_vars.values()): + if isinstance(value, _Environment): + yield value, module_file + + +def _declaring_file(env: Environment) -> Path | None: + """The file of a loaded module holding *env* in its globals, else None.""" + return next((Path(file) for live, file in _live_envs() if live is env), None) + + +def _declared_env(name: str) -> Environment | None: + """The one live env named *name*, else None; two distinct ones raise. + + The same instance re-exported across modules is one match; distinct envs + claiming one name are ambiguous. + """ + matches = {id(env): env for env, _ in _live_envs() if env.name == name} + if len(matches) > 1: + files = sorted({file for env, file in _live_envs() if env.name == name}) + raise ValueError( + f"env name {name!r} is declared by multiple live environments " + f"({', '.join(files)}); pass runtime= explicitly — the exact " + "instance disambiguates: runtime=LocalRuntime(env)" + ) + return next(iter(matches.values()), None) + + +def _declared_names(source: Path) -> set[str]: + """Env names a ``.py`` source (file or directory) declares at module level.""" + from hud.environment.env import Environment as _Environment + from hud.utils.modules import iter_modules + + return { + value.name + for module in iter_modules(source) + for value in vars(module).values() + if isinstance(value, _Environment) + } class SubprocessRuntime: diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index 9da2e59e2..691854fde 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -14,18 +14,16 @@ import asyncio import json import logging -import sys import uuid from pathlib import Path from typing import TYPE_CHECKING, Any -from hud.environment.env import Environment from hud.telemetry import flush from hud.utils.platform import PlatformClient from .job import Job, job_enter from .run import rollout -from .runtime import HostedRuntime, HUDRuntime, LocalRuntime +from .runtime import HostedRuntime, HUDRuntime, LocalRuntime, _declared_env, _declared_names from .sync import fetch_taskset_tasks, resolve_taskset_id if TYPE_CHECKING: @@ -40,50 +38,6 @@ logger = logging.getLogger("hud.eval.taskset") -def _declared_env(name: str) -> Environment | None: - """A live env named *name* declared in a loaded module's globals, else None. - - The in-memory counterpart of :func:`~hud.environment.load_environment`'s - file scan: an env declared at module top level in an imported module can - be served fresh from its file (``LocalRuntime`` locates it). Distinct - envs claiming one name is ambiguous and raises; the same instance - re-exported across modules is one match. Envs not in any file-backed - module's globals (constructed inside a function, a notebook cell) - return None. - """ - matches: dict[int, tuple[Environment, str]] = {} - for module in list(sys.modules.values()): - module_file = getattr(module, "__file__", None) - module_vars = getattr(module, "__dict__", None) - if not module_file or not isinstance(module_vars, dict): - continue - for value in list(module_vars.values()): - if isinstance(value, Environment) and value.name == name: - matches.setdefault(id(value), (value, module_file)) - if not matches: - return None - if len(matches) > 1: - files = sorted({file for _, file in matches.values()}) - raise ValueError( - f"env name {name!r} is declared by multiple live environments " - f"({', '.join(files)}); pass runtime= explicitly — the exact " - "instance disambiguates: runtime=LocalRuntime(env)" - ) - return next(iter(matches.values()))[0] - - -def _declared_names(source: Path) -> set[str]: - """Env names a ``.py`` source (file or directory) declares at module level.""" - from hud.utils.modules import iter_modules - - return { - value.name - for module in iter_modules(source) - for value in vars(module).values() - if isinstance(value, Environment) - } - - def _job_name(taskset_name: str, tasks: list[Task], group: int) -> str: suffix = f" ({group} times)" if group > 1 else "" if len(tasks) == 1: From 17f77c91fafd72bb8fb55763d24de767cfd84772 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:54:53 -0700 Subject: [PATCH 8/9] fix(eval): re-exports never claim envs; empty tasksets need no placement; source dir stays importable Addresses the second round of bot findings on #484. One discriminator fixes the two re-export findings: a fresh execution of a source yields new instances for envs it declares and the already-live instance for envs it merely imports. The origin probe counts only newly-created instances, so a tasks file re-exporting its env falls through to the env's real module and every rollout is rebuilt fresh; the live-env pointer validates candidate files the same way, skipping package __init__ re-exports (tried last) and anything whose standalone import fails or returns the live instance. An empty taskset schedules nothing, so it resolves no placement (it previously raised). And the source directory stays on sys.path for the whole acquisition, not just the initial import, so a template can lazily import a sibling module at run time as it could under the child-process runtime (insert-and-remove one entry per acquisition, balanced under concurrency). --- hud/eval/runtime.py | 85 ++++++++++++++++++------- hud/eval/taskset.py | 6 +- hud/eval/tests/test_local_runtime.py | 95 ++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 26 deletions(-) diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index e9a589ab3..a05e16ab1 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -199,18 +199,21 @@ def __init__( # at acquisition with the cause named instead of "unknown task" later. expected_templates: frozenset[str] = frozenset() if isinstance(source, _Environment): - file = _declaring_file(source) + file = _declaring_file(source, env or source.name) if file is None: raise TypeError( - f"LocalRuntime: env {source.name!r} is not declared at module " - "level in an importable file, so it cannot be rebuilt fresh per " - "rollout; pass its constructor instead: " + f"LocalRuntime: env {source.name!r} is not rebuilt by importing " + "any file this process has loaded (constructed in a function or " + "notebook cell, or declared inside a package using relative " + "imports); pass its constructor instead: " "LocalRuntime(lambda task: )" ) expected_templates = frozenset(source.tasks) source, env = file, env or source.name + self._source_dir: Path | None = None if isinstance(source, (str, Path)): path, pinned = Path(source).resolve(), env + self._source_dir = path if path.is_dir() else path.parent from hud.environment import load_environment def _load(task: Task) -> _Environment: @@ -242,21 +245,32 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: if task.runtime_config is not None: raise ValueError("LocalRuntime does not support task runtime_config") + # The source dir stays importable for the whole acquisition, not just + # the initial import, so a template can lazily import a sibling + # module at run time (as it could under the child-process runtime). + # Always insert-and-remove one entry: balanced under concurrency. + if self._source_dir is not None: + sys.path.insert(0, str(self._source_dir)) try: - env = self._build(task) - except RuntimeError as e: - # The source ran an event loop at import — usually an unguarded - # top-level run call; name the actual mistake. - if "running event loop" not in str(e): - raise - raise RuntimeError( - "the env source ran async code while being imported to place a " - 'rollout — guard top-level run calls with `if __name__ == "__main__":`' - ) from e - if not isinstance(env, _Environment): - raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") - async with _local(env, ready_timeout=self.ready_timeout) as runtime: - yield runtime + try: + env = self._build(task) + except RuntimeError as e: + # The source ran an event loop at import — usually an unguarded + # top-level run call; name the actual mistake. + if "running event loop" not in str(e): + raise + raise RuntimeError( + "the env source ran async code while being imported to place a " + 'rollout — guard top-level run calls with `if __name__ == "__main__":`' + ) from e + if not isinstance(env, _Environment): + raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") + async with _local(env, ready_timeout=self.ready_timeout) as runtime: + yield runtime + finally: + if self._source_dir is not None: + with contextlib.suppress(ValueError): + sys.path.remove(str(self._source_dir)) def _live_envs() -> Iterator[tuple[Environment, str]]: @@ -280,9 +294,28 @@ def _live_envs() -> Iterator[tuple[Environment, str]]: yield value, module_file -def _declaring_file(env: Environment) -> Path | None: - """The file of a loaded module holding *env* in its globals, else None.""" - return next((Path(file) for live, file in _live_envs() if live is env), None) +def _declaring_file(env: Environment, name: str) -> Path | None: + """A file whose fresh import re-declares *env*, else None. + + Candidate files hold the instance in their module globals, but a holder + may be a re-exporter (``from .env import env`` in a package + ``__init__``, a tasks file re-exporting its env): validate each by + loading it fresh — a declarer yields a *new* instance under *name*, a + re-exporter yields the same live one (or fails to import standalone). + ``__init__.py`` holders are tried last. + """ + from hud.environment import load_environment + + candidates = dict.fromkeys(Path(file) for live, file in _live_envs() if live is env) + for file in sorted(candidates, key=lambda f: f.name == "__init__.py"): + try: + probe = load_environment(file, name=name) + except Exception as e: + logger.debug("candidate %s does not rebuild env %r: %s", file, name, e) + continue + if probe is not env: + return file + return None def _declared_env(name: str) -> Environment | None: @@ -303,15 +336,21 @@ def _declared_env(name: str) -> Environment | None: def _declared_names(source: Path) -> set[str]: - """Env names a ``.py`` source (file or directory) declares at module level.""" + """Env names a ``.py`` source (file or directory) itself declares. + + A fresh execution of the source yields *new* instances for envs it + declares; an env it merely imports is the already-live one and does not + count — importing the source again could not rebuild it. + """ from hud.environment.env import Environment as _Environment from hud.utils.modules import iter_modules + live = {id(env) for env, _ in _live_envs()} return { value.name for module in iter_modules(source) for value in vars(module).values() - if isinstance(value, _Environment) + if isinstance(value, _Environment) and id(value) not in live } diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index 691854fde..77ec8a52f 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -303,12 +303,12 @@ async def run( # taskset runs where it came from; rows naming envs declared in # imported modules serve each fresh from its file; anything else is # an error naming the forms to pass. - if runtime is None: - runtime = self._resolve_placement() - placement = runtime + # An empty taskset schedules nothing, so it needs no placement. + placement = runtime if runtime is not None or not task_list else self._resolve_placement() sem = asyncio.Semaphore(max_concurrent) if max_concurrent else None async def _run(task: Task, group_id: str) -> Run: + assert placement is not None # only reached when tasks were expanded if isinstance(placement, HostedRuntime): return await placement.run(task, agent, job_id=job_id, group_id=group_id) return await rollout( diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py index 5e33ab7eb..fd8ec4240 100644 --- a/hud/eval/tests/test_local_runtime.py +++ b/hud/eval/tests/test_local_runtime.py @@ -272,6 +272,101 @@ async def test_single_file_taskset_never_drags_in_a_same_named_sibling(tmp_path) assert [run.reward for run in job.runs] == [1.0] +async def test_empty_taskset_runs_without_a_placement() -> None: + job = await Taskset("empty", []).run(_FnAgent(_solve_add)) + + assert job.runs == [] + + +async def test_reexporting_tasks_module_does_not_claim_the_env( + tmp_path, monkeypatch, request +) -> None: + # tasks.py re-exports the env object alongside the factory: the origin + # must not claim it (re-import of tasks.py would reuse the cached env + # module) — each rollout rebuilds the env from its real file instead. + env_dir = tmp_path / "pkg" + env_dir.mkdir() + (env_dir / "sums_reexp_envmod.py").write_text( + "from hud import Environment\n\n" + "LOADS = []\n" + 'env = Environment("sums")\n\n\n' + '@env.template(id="add")\nasync def add(a: int, b: int):\n' + " LOADS.append(1)\n" + ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n' + " yield 1.0 if answer == str(a + b) else 0.0\n", + encoding="utf-8", + ) + tasks_dir = tmp_path / "tasks" + tasks_dir.mkdir() + (tasks_dir / "tasks.py").write_text( + "from sums_reexp_envmod import add, env\n\ntasks = [add(a=2, b=3)]\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(env_dir)) + request.addfinalizer(lambda: sys.modules.pop("sums_reexp_envmod", None)) + + def _solve(prompt: str) -> str: + _, a, b, loads = prompt.split(":") + assert loads == "1" # a fresh env module per rollout, not the cached one + return str(int(a) + int(b)) + + taskset = Taskset.from_module(tasks_dir / "tasks.py") + job = await taskset.run(_FnAgent(_solve), group=2) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + + +async def test_package_reexported_env_pointer_uses_the_declaring_submodule( + tmp_path, monkeypatch, request +) -> None: + pkg = tmp_path / "sums_pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("from .env_mod import env\n", encoding="utf-8") + (pkg / "env_mod.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + request.addfinalizer( + lambda: [sys.modules.pop(m, None) for m in ("sums_pkg", "sums_pkg.env_mod")] + ) + import importlib + + sums_pkg = importlib.import_module("sums_pkg") + + run = await rollout( + Task(env="sums", id="add", args={"a": 2, "b": 3}), + _FnAgent(_solve_add), + runtime=LocalRuntime(sums_pkg.env), + ) + + assert run.reward == 1.0 + + +async def test_template_can_lazily_import_a_sibling_module(tmp_path) -> None: + (tmp_path / "lazy_helper.py").write_text("ANSWER_SUFFIX = ':ok'\n", encoding="utf-8") + (tmp_path / "env.py").write_text( + "from hud import Environment\n\n" + 'env = Environment("sums")\n\n\n' + '@env.template(id="add")\nasync def add(a: int, b: int):\n' + " import lazy_helper\n" + ' answer = yield f"add:{a}:{b}{lazy_helper.ANSWER_SUFFIX}"\n' + " yield 1.0 if answer == str(a + b) else 0.0\n", + encoding="utf-8", + ) + + def _solve(prompt: str) -> str: + _, a, b, ok = prompt.split(":") + assert ok == "ok" + return str(int(a) + int(b)) + + job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( + _FnAgent(_solve), + runtime=LocalRuntime(tmp_path / "env.py"), + group=2, + max_concurrent=2, + ) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + + # ─── seam defenses ───────────────────────────────────────────────────── From b8be4a2710489af2eb4c61a985611e21c455cb1a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:00:57 -0700 Subject: [PATCH 9/9] docs(eval): the no-placement error names the origin-preserving path for extracted rows --- hud/eval/taskset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index 77ec8a52f..2ef0d223a 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -234,7 +234,9 @@ def _resolve_placement(self) -> Provider | HUDRuntime: f"no placement for env(s) {', '.join(missing) or ''}: pass runtime= — " 'LocalRuntime("env.py") (a source file), LocalRuntime(env) (a live env), ' "LocalRuntime(build) (a (task) -> Environment constructor), Runtime(url) " - "(a served substrate), or HUDRuntime() (your deployed env)" + "(a served substrate), or HUDRuntime() (your deployed env). A row taken " + "from a loaded taskset keeps its placement when run through it: " + 'taskset.filter(["slug"]).run(...)' ) async def run(