diff --git a/docs/reference/migrations.rst b/docs/reference/migrations.rst index 7d33ebad3..eb97fba2d 100644 --- a/docs/reference/migrations.rst +++ b/docs/reference/migrations.rst @@ -3,6 +3,17 @@ Migration Guide This page can help when migrating Taskgraph across major versions. +23.x -> 24.x +------------ + +* You must now pass in `--allow-parameter-override` into Decision tasks that + need to load the `try_task_config.json` file at the root of the repo. +* `taskgraph.parameters.Parameters.is_try()` is removed. Either in-line the old + method to a utility or use alternative logic. +* Level 1 artifacts' default expiry changed from "1 year" to "28 days". If + needed, set `task-expires-after` in `taskcluster/config.yml` to adjust the + default back. + 22.x -> 23.x ------------ diff --git a/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py b/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py index ac1bee702..53d762943 100644 --- a/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +++ b/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py @@ -123,9 +123,6 @@ def graph_config(datadir): class FakeParameters(dict): strict = True - def is_try(self): - return self["level"] != 3 - def file_url(self, path, pretty=False): return path diff --git a/src/taskgraph/decision.py b/src/taskgraph/decision.py index 4a35ca2d1..a27eda672 100644 --- a/src/taskgraph/decision.py +++ b/src/taskgraph/decision.py @@ -252,31 +252,27 @@ def get_decision_parameters(graph_config, options): assert callable(decision_params) decision_params(graph_config, parameters) - if options.get("try_task_config_file"): - task_config_file = os.path.abspath(options.get("try_task_config_file")) - else: - # if try_task_config.json is present, load it - task_config_file = os.path.join(os.getcwd(), "try_task_config.json") - - # load try settings - if ("try" in project and options["tasks_for"] == "hg-push") or options[ - "tasks_for" - ] == "github-pull-request": - set_try_config(parameters, task_config_file) - - # load extra parameters from vcs note if able - note_ref = "refs/notes/decision-parameters" - if options.get("allow_parameter_override") and ( - note_params := repo.get_note(note_ref, parameters["head_repository"]) - ): - try: - note_params = json.loads(note_params) - logger.info( - f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}" - ) - parameters.update(note_params) - except ValueError as e: - raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e + # load extra parameters from file or vcs note if able + if options.get("allow_parameter_override"): + note_ref = "refs/notes/decision-parameters" + if options.get("try_task_config_file"): + task_config_file = os.path.abspath(options.get("try_task_config_file")) + else: + # if try_task_config.json is present, load it + task_config_file = os.path.join(os.getcwd(), "try_task_config.json") + + if os.path.isfile(task_config_file): + set_try_config(parameters, task_config_file) + + elif note_params := repo.get_note(note_ref, parameters["head_repository"]): + try: + note_params = json.loads(note_params) + logger.info( + f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}" + ) + parameters.update(note_params) + except ValueError as e: + raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e result = Parameters(**parameters) result.check() @@ -284,23 +280,22 @@ def get_decision_parameters(graph_config, options): def set_try_config(parameters, task_config_file): - if os.path.isfile(task_config_file): - logger.info(f"using try tasks from {task_config_file}") - with open(task_config_file) as fh: - task_config = json.load(fh) - task_config_version = task_config.pop("version") - if task_config_version == 2: - validate_schema( - try_task_config_schema_v2, - task_config, - "Invalid v2 `try_task_config.json`.", - ) - parameters.update(task_config["parameters"]) - return - else: - raise Exception( - f"Unknown `try_task_config.json` version: {task_config_version}" - ) + logger.info(f"using try tasks from {task_config_file}") + with open(task_config_file) as fh: + task_config = json.load(fh) + task_config_version = task_config.pop("version") + if task_config_version == 2: + validate_schema( + try_task_config_schema_v2, + task_config, + "Invalid v2 `try_task_config.json`.", + ) + parameters.update(task_config["parameters"]) + return + else: + raise Exception( + f"Unknown `try_task_config.json` version: {task_config_version}" + ) def write_artifact(filename, data): diff --git a/src/taskgraph/parameters.py b/src/taskgraph/parameters.py index 64f6065c2..db51e48b2 100644 --- a/src/taskgraph/parameters.py +++ b/src/taskgraph/parameters.py @@ -273,13 +273,6 @@ def __getitem__(self, k): except KeyError: raise KeyError(f"taskgraph parameter {k!r} not found") - def is_try(self): - """ - Determine whether this graph is being built on a try project or for - `mach try fuzzy`. - """ - return "try" in self["project"] or self["tasks_for"] == "github-pull-request" - @property def moz_build_date(self): # XXX self["moz_build_date"] is left as a string because: diff --git a/src/taskgraph/transforms/docker_image.py b/src/taskgraph/transforms/docker_image.py index 643be20e5..9363de2d0 100644 --- a/src/taskgraph/transforms/docker_image.py +++ b/src/taskgraph/transforms/docker_image.py @@ -130,7 +130,7 @@ def fill_template(config, tasks): "artifact_prefix": "public", }, "always-target": True, - "expires-after": expires if config.params.is_try() else "1 year", + "expires-after": expires if config.params["level"] == "1" else "1 year", "scopes": [], "run-on-projects": [], "worker-type": "images", diff --git a/src/taskgraph/transforms/task.py b/src/taskgraph/transforms/task.py index 1c3ceb972..28b85e180 100644 --- a/src/taskgraph/transforms/task.py +++ b/src/taskgraph/transforms/task.py @@ -457,12 +457,10 @@ def build_docker_worker_payload(config, task, task_def): else: suffix = cache_version - skip_untrusted = config.params.is_try() or level == 1 - for cache in worker["caches"]: # Some caches aren't enabled in environments where we can't # guarantee certain behavior. Filter those out. - if cache.get("skip-untrusted") and skip_untrusted: + if cache.get("skip-untrusted") and level == 1: continue name = "{trust_domain}-level-{level}-{name}-{suffix}".format( @@ -484,7 +482,7 @@ def build_docker_worker_payload(config, task, task_def): if run_task and worker.get("volumes"): payload["env"]["TASKCLUSTER_VOLUMES"] = ";".join(sorted(worker["volumes"])) - if payload.get("cache") and skip_untrusted: # type: ignore + if payload.get("cache") and level == 1: # type: ignore payload["env"]["TASKCLUSTER_UNTRUSTED_CACHES"] = "1" if features: @@ -999,7 +997,7 @@ def build_task(config, tasks): if "expires-after" not in task: task["expires-after"] = ( config.graph_config._config.get("task-expires-after", "28 days") - if config.params.is_try() + if config.params["level"] == "1" else "1 year" )