From 7bf4d3fbd7f5911cfaee63a4b55eb5b0086db3ea Mon Sep 17 00:00:00 2001 From: George Weale Date: Tue, 28 Jul 2026 17:09:23 +0000 Subject: [PATCH 1/2] docs(evaluate): fix conformance CLI, defaults, optimizer imports --- docs/evaluate/criteria.md | 17 ++++++++-------- docs/evaluate/environment_simulation.md | 2 +- docs/evaluate/index.md | 26 ++++++++++++++++--------- docs/evaluate/user-sim.md | 8 ++++++-- docs/optimize/index.md | 19 ++++++++++++++---- 5 files changed, 47 insertions(+), 25 deletions(-) diff --git a/docs/evaluate/criteria.md b/docs/evaluate/criteria.md index 29d5bea45c..d60a3957df 100644 --- a/docs/evaluate/criteria.md +++ b/docs/evaluate/criteria.md @@ -231,10 +231,9 @@ Example `EvalConfig` entry: "final_response_match_v2": { "threshold": 0.8, "judge_model_options": { - "judge_model": "gemini-flash-latest", - "num_samples": 5 - } - } + "judge_model": "gemini-flash-latest", + "num_samples": 5 + } } } } @@ -337,7 +336,7 @@ The merged rubric list passed to the judge is the union of the criterion-level l #### Notes On Rubrics -- Rubrics on `EvalConfig.criteria["rubric_based_final_response_quality_v1"].rubrics` **must be non-empty** — `RubricBasedEvaluator` asserts this at init time. +- The effective rubric list **must be non-empty**, otherwise `RubricBasedEvaluator` raises a `ValueError` at evaluation time. The criterion-level list on `EvalConfig.criteria["rubric_based_final_response_quality_v1"].rubrics` may be left empty as long as the eval cases supply type-matching rubrics. - Rubrics on `EvalCase.rubrics` are *additive* on top of the criterion-level list, not a replacement. The effective rubric set passed to the judge is the union of both. - Rubrics supplied per-case via `EvalCase.rubrics` are filtered by `type`: only those whose `type` is `"FINAL_RESPONSE_QUALITY"` are merged in. Criterion-level rubrics in `EvalConfig` are **not** filtered by `type`. @@ -439,7 +438,7 @@ The merged rubric list passed to the judge is the union of the criterion-level l #### Notes On Rubrics -- Rubrics on `EvalConfig.criteria["rubric_based_tool_use_quality_v1"].rubrics` **must be non-empty** — `RubricBasedEvaluator` asserts this at init time. +- The effective rubric list **must be non-empty**, otherwise `RubricBasedEvaluator` raises a `ValueError` at evaluation time. The criterion-level list on `EvalConfig.criteria["rubric_based_tool_use_quality_v1"].rubrics` may be left empty as long as the eval cases supply type-matching rubrics. - Rubrics on `EvalCase.rubrics` are *additive* on top of the criterion-level list, not a replacement. The effective rubric set passed to the judge is the union of both. - Rubrics supplied per-case via `EvalCase.rubrics` are filtered by `type`: only those whose `type` is `"TOOL_USE_QUALITY"` are merged in. Criterion-level rubrics in `EvalConfig` are **not** filtered by `type`. @@ -538,7 +537,7 @@ The merged rubric list passed to the judge is the union of the criterion-level l #### Notes On Rubrics -- Rubrics on `EvalConfig.criteria["rubric_based_multi_turn_trajectory_quality_v1"].rubrics` **must be non-empty** — `RubricBasedEvaluator` asserts this at init time. +- The effective rubric list **must be non-empty**, otherwise `RubricBasedEvaluator` raises a `ValueError` at evaluation time. The criterion-level list on `EvalConfig.criteria["rubric_based_multi_turn_trajectory_quality_v1"].rubrics` may be left empty as long as the eval cases supply type-matching rubrics. - Rubrics on `EvalCase.rubrics` are *additive* on top of the criterion-level list, not a replacement. The effective rubric set passed to the judge is the union of both. - Rubrics supplied per-case via `EvalCase.rubrics` are filtered by `type`: only those whose `type` is `"TRAJECTORY_QUALITY"` are merged in. Criterion-level rubrics in `EvalConfig` are **not** filtered by `type`. @@ -592,8 +591,8 @@ Example `EvalConfig` entry: "hallucinations_v1": { "threshold": 0.8, "judge_model_options": { - "judge_model": "gemini-flash-latest", - }, + "judge_model": "gemini-flash-latest" + }, "evaluate_intermediate_nl_responses": true } } diff --git a/docs/evaluate/environment_simulation.md b/docs/evaluate/environment_simulation.md index 3a62e41aae..ca1fa09b3f 100644 --- a/docs/evaluate/environment_simulation.md +++ b/docs/evaluate/environment_simulation.md @@ -138,7 +138,7 @@ The top-level configuration object. Field | Type | Default | Description :------------------------------- | :--------------------------- | :------------------- | :---------- `tool_simulation_configs` | `List[ToolSimulationConfig]` | required | One entry per tool to simulate. Must not be empty, and tool names must be unique. -`simulation_model` | `str` | `"gemini-flash-latest"` | The LLM used for tool connection analysis and mock response generation. +`simulation_model` | `str` | `"gemini-2.5-flash"` | The LLM used for tool connection analysis and mock response generation. `simulation_model_configuration` | `GenerateContentConfig` | thinking enabled | LLM generation config for internal simulator calls. `environment_data` | `str \| None` | `None` | Optional environment context (e.g., a JSON database snapshot) passed to mock strategies to generate more realistic responses. `tracing` | `str \| None` | `None` | Tracing data (e.g., a prior agent run trace in JSON string format) to provide historical context. diff --git a/docs/evaluate/index.md b/docs/evaluate/index.md index 9e619a40d6..07d734323c 100644 --- a/docs/evaluate/index.md +++ b/docs/evaluate/index.md @@ -338,11 +338,13 @@ tests In your target test folder, create a `spec.yaml` file. This file outlines the initial conditions, configurations, and user prompts that the agent will execute during the baseline recording and subsequent conformance runs. Ensure your file matches the following basic schema, this is an example only: -``` -# Example spec.yaml for a Weather Agent name: "current_weather_check" description: -"Verifies the agent correctly identifies location and calls the weather tool." -user_prompts: - "What's the temperature in San Francisco right now?" expected_tools: - - "get_weather_api" +```yaml +# Example spec.yaml for a Weather Agent. +# The test case name and category are inferred from the folder structure. +description: "Verifies the agent correctly identifies location and calls the weather tool." +agent: "weather_agent" +user_messages: + - text: "What's the temperature in San Francisco right now?" ``` #### Automate the baseline @@ -358,9 +360,11 @@ adk web -v --extra_plugins=google.adk.cli.plugins.recordings_plugin.RecordingsPl 2. Next, open a new terminal window and tell ADK to create the baseline files based on your spec.yaml: ```shell -adk conformance create tests/category/test_name +adk conformance record tests/category/test_name none ``` + The trailing streaming-mode argument is required. Use `none` to record `generated-recordings.yaml` and `generated-session.yaml`, or `sse` to record `generated-recordings-sse.yaml` and `generated-session-sse.yaml` instead. The `bidi` mode is not supported for recording. + This automatically runs the scenario, records all the interactions, and saves the generated-recordings.yaml and generated-session.yaml files exactly where they need to be. Once these baseline files are locked in, your setup is complete, and the directory is ready to be targeted by `adk conformance` in either **Replay** or **Live** mode. @@ -387,6 +391,8 @@ Here is a summary of all the available criteria: quality based on custom rubrics. * **rubric_based_tool_use_quality_v1**: LLM-judged tool usage quality based on custom rubrics. +* **rubric_based_multi_turn_trajectory_quality_v1**: LLM-judged multi-turn + trajectory quality based on custom rubrics. * **hallucinations_v1**: LLM-judged groundedness of agent response against context. * **safety_v1**: Safety/harmlessness of agent response. @@ -462,7 +468,9 @@ Choose criteria based on your evaluation goals: In addition, criteria which require information on expected agent tool use and/or responses are not supported in combination with [User Simulation](./user-sim.md). -Currently, only the `hallucinations_v1` and `safety_v1` criteria support such evals. +The affected criteria are `tool_trajectory_avg_score`, `response_match_score` +and `final_response_match_v2`. Every other criterion supports user simulation, +as shown in the support column of [Evaluation Criteria](./criteria.md). ### User simulation @@ -582,7 +590,7 @@ Here is the command: ```shell adk eval \ \ - \ + ... \ [--config_file_path=] \ [--print_detailed_results] ``` @@ -598,7 +606,7 @@ adk eval \ Here are the details for each command line argument: * `AGENT_MODULE_FILE_PATH`: The path to the `__init__.py` file that contains a module by the name "agent". "agent" module contains a `root_agent`. -* `EVAL_SET_FILE_PATH`: The path to evaluations file(s). You can specify one or more eval set file paths. For each file, all evals will be run by default. If you want to run only specific evals from a eval set, first create a comma separated list of eval names and then add that as a suffix to the eval set file name, demarcated by a colon `:` . +* `EVAL_SET_FILE_PATH_OR_ID`: The path to evaluations file(s), or the id of an eval set managed by ADK (as created by `adk eval_set create`). You can specify one or more of either, but you cannot mix file paths and eval set ids in the same command. For each eval set, all evals will be run by default. If you want to run only specific evals from a eval set, first create a comma separated list of eval names and then add that as a suffix to the eval set file name or id, demarcated by a colon `:` . * For example: `sample_eval_set_file.json:eval_1,eval_2,eval_3` `This will only run eval_1, eval_2 and eval_3 from sample_eval_set_file.json` * `CONFIG_FILE_PATH`: The path to the config file. diff --git a/docs/evaluate/user-sim.md b/docs/evaluate/user-sim.md index ed75848151..d3256bcdd7 100644 --- a/docs/evaluate/user-sim.md +++ b/docs/evaluate/user-sim.md @@ -194,7 +194,7 @@ The below `EvalConfig` shows the default user simulator configuration: # same as before }, "user_simulator_config": { - "model": "gemini-flash-latest", + "model": "gemini-2.5-flash", "model_configuration": { "thinking_config": { "include_thoughts": true, @@ -213,7 +213,11 @@ The below `EvalConfig` shows the default user simulator configuration: * `max_allowed_invocations`: The maximum user-agent interactions allowed before the conversation is forcefully terminated. This should be set to be greater than the longest reasonable user-agent interaction in your - `EvalSet`. + `EvalSet`. The initial fixed prompt counts as an invocation. Set it to `-1` + for no limit (not recommended). +* `include_function_calls`: Optional. Whether to include function calls and + responses in the conversation history prompt given to the user simulator. + Defaults to `false`. * `custom_instructions`: Optional. Overrides the default instructions for the user simulator. The instruction string must contain the following formatting placeholders using diff --git a/docs/optimize/index.md b/docs/optimize/index.md index ea9e7cd10c..afc310231f 100644 --- a/docs/optimize/index.md +++ b/docs/optimize/index.md @@ -237,12 +237,16 @@ additional metrics collected during optimization. Note: The `GEPARootAgentPromptOptimizer` does not improve any sub-agents, agent tools, skills, or any other aspect of the root agent. +Note: The `GEPARootAgentPromptOptimizer` is experimental. +It emits a warning when constructed, and its API may change or be removed +without notice. + You can configure the `GEPARootAgentPromptOptimizer` with a `GEPARootAgentPromptOptimizerConfig` that contains the following fields: * `optimizer_model` (optional): The model used to analyze evaluation results and optimize the agent. -Defaults to `"gemini-flash-latest"`. +Defaults to `"gemini-2.5-flash"`. * `model_configuration` (optional): The configuration for the optimizer model. Defaults to a config with a 10K token thinking budget. * `max_metric_calls` (optional): The maximum number of evaluations to run during @@ -274,6 +278,10 @@ additional metrics collected during optimization. Note: The `GEPARootAgentOptimizer` does not improve any sub-agents or agent tools. +Note: The `GEPARootAgentOptimizer` is experimental. +It emits a warning when constructed, and its API may change or be removed +without notice. + You can configure the `GEPARootAgentOptimizer` with a `GEPARootAgentOptimizerConfig` that contains the following fields: @@ -315,15 +323,18 @@ Configure the behavior of the loop by passing a `SimplePromptOptimizerConfig` in | Parameter | Type | Default | Description | | :---- | :---- | :---- | :---- | -| `num_iterations` | int | *Required* | The total number of optimization rounds to execute. | -| `batch_size` | int | *Required* | The number of evaluation sample cases processed by the sampler during each individual iteration. | +| `num_iterations` | int | `10` | The total number of optimization rounds to execute. | +| `batch_size` | int | `5` | The number of evaluation sample cases processed by the sampler during each individual iteration. | +| `optimizer_model` | str | `"gemini-2.5-flash"` | The model used to critique the current prompt and generate the next one. | +| `model_configuration` | GenerateContentConfig | thinking budget of 10K tokens | The configuration for the optimizer model. | #### Implementation Example Once your configuration is defined, run the optimization with: ```python -from google.adk.optimization import SimplePromptOptimizer, SimplePromptOptimizerConfig +from google.adk.optimization.simple_prompt_optimizer import SimplePromptOptimizer +from google.adk.optimization.simple_prompt_optimizer import SimplePromptOptimizerConfig # Define your Agent and Sampler first... From a10ba6a5e4f623e1ffc38cb736c5f30eef6746f0 Mon Sep 17 00:00:00 2001 From: George Weale Date: Thu, 30 Jul 2026 00:22:20 +0000 Subject: [PATCH 2/2] docs(evaluate): fix App construction, agent path type, user-sim coverage --- docs/evaluate/criteria.md | 1 + docs/evaluate/environment_simulation.md | 3 ++- docs/evaluate/index.md | 8 ++++---- docs/evaluate/user-sim.md | 3 ++- docs/optimize/index.md | 4 ++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/evaluate/criteria.md b/docs/evaluate/criteria.md index d60a3957df..584554e860 100644 --- a/docs/evaluate/criteria.md +++ b/docs/evaluate/criteria.md @@ -11,6 +11,7 @@ Criterion | Description :--------------------------------------- | :-------------------------------------------------------- | :-------------- | :--------------- | :------------- | :---------------------------------------- `tool_trajectory_avg_score` | Exact match of tool call trajectory | Yes | No | No | No `response_match_score` | ROUGE-1 similarity to reference response | Yes | No | No | No +`response_evaluation_score` | Vertex AI coherence score for the agent response | Yes | No | Yes | No `final_response_match_v2` | LLM-judged semantic match to reference response | Yes | No | Yes | No `rubric_based_final_response_quality_v1` | LLM-judged final response quality based on custom rubrics | No | Yes | Yes | Yes `rubric_based_tool_use_quality_v1` | LLM-judged tool usage quality based on custom rubrics | No | Yes | Yes | Yes diff --git a/docs/evaluate/environment_simulation.md b/docs/evaluate/environment_simulation.md index ca1fa09b3f..2f259a5eef 100644 --- a/docs/evaluate/environment_simulation.md +++ b/docs/evaluate/environment_simulation.md @@ -121,7 +121,8 @@ config = EnvironmentSimulationConfig( ) app = App( - agent=my_agent, + name="my_app", + root_agent=my_agent, plugins=[EnvironmentSimulationFactory.create_plugin(config)], ) ``` diff --git a/docs/evaluate/index.md b/docs/evaluate/index.md index 07d734323c..1673286fe9 100644 --- a/docs/evaluate/index.md +++ b/docs/evaluate/index.md @@ -468,9 +468,9 @@ Choose criteria based on your evaluation goals: In addition, criteria which require information on expected agent tool use and/or responses are not supported in combination with [User Simulation](./user-sim.md). -The affected criteria are `tool_trajectory_avg_score`, `response_match_score` -and `final_response_match_v2`. Every other criterion supports user simulation, -as shown in the support column of [Evaluation Criteria](./criteria.md). +The affected criteria include `tool_trajectory_avg_score`, `response_match_score` +and `final_response_match_v2`. Support for user simulation is listed per +criterion in the support column of [Evaluation Criteria](./criteria.md). ### User simulation @@ -605,7 +605,7 @@ adk eval \ Here are the details for each command line argument: -* `AGENT_MODULE_FILE_PATH`: The path to the `__init__.py` file that contains a module by the name "agent". "agent" module contains a `root_agent`. +* `AGENT_MODULE_FILE_PATH`: The path to the agent directory (not a file) whose `__init__.py` exposes a module by the name "agent". "agent" module contains a `root_agent`. * `EVAL_SET_FILE_PATH_OR_ID`: The path to evaluations file(s), or the id of an eval set managed by ADK (as created by `adk eval_set create`). You can specify one or more of either, but you cannot mix file paths and eval set ids in the same command. For each eval set, all evals will be run by default. If you want to run only specific evals from a eval set, first create a comma separated list of eval names and then add that as a suffix to the eval set file name or id, demarcated by a colon `:` . * For example: `sample_eval_set_file.json:eval_1,eval_2,eval_3` `This will only run eval_1, eval_2 and eval_3 from sample_eval_set_file.json` diff --git a/docs/evaluate/user-sim.md b/docs/evaluate/user-sim.md index d3256bcdd7..d8901f6b06 100644 --- a/docs/evaluate/user-sim.md +++ b/docs/evaluate/user-sim.md @@ -201,7 +201,8 @@ The below `EvalConfig` shows the default user simulator configuration: "thinking_budget": 10240 } }, - "max_allowed_invocations": 20 + "max_allowed_invocations": 20, + "include_function_calls": false } } ``` diff --git a/docs/optimize/index.md b/docs/optimize/index.md index afc310231f..35459ae0a6 100644 --- a/docs/optimize/index.md +++ b/docs/optimize/index.md @@ -166,8 +166,8 @@ Optimized root agent instructions: adk optimize [OPTIONS] AGENT_MODULE_FILE_PATH ``` -* `AGENT_MODULE_FILE_PATH`: The path to the `__init__.py` file that contains a -module by the name `agent`. +* `AGENT_MODULE_FILE_PATH`: The path to the agent directory (not a file) whose +`__init__.py` exposes a module by the name `agent`. The `agent` module must contain a `root_agent`. For an example of a valid setup, examine the [`hello_world`](https://github.com/google/adk-python/tree/main/contributing/samples/core/hello_world)