diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 7256937a87..407fd1cb48 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -38,6 +38,7 @@ common --deleted_packages=tests/integration/pip_parse common --deleted_packages=tests/integration/pip_parse/empty common --deleted_packages=tests/integration/pip_parse_isolated common --deleted_packages=tests/integration/py_cc_toolchain_registered +common --deleted_packages=tests/integration/runtime_manifests common --deleted_packages=tests/integration/toolchain_target_settings common --deleted_packages=tests/integration/uv_lock common --deleted_packages=tests/modules/another_module diff --git a/CHANGELOG.md b/CHANGELOG.md index 33adfa7a60..40aed9e5d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-added} ### Added +* (toolchains) Support dynamically fetching and registering Python runtimes + from a python-build-standalone manifest file using + `python.override(add_runtime_manifest_urls = ..., runtime_manifest_sha = ...)`. * (toolchain) Added {obj}`python.override.toolchain_target_settings` to allow adding `config_setting` labels to all registered toolchains. * (windows) Full venv support for Windows is available. Set diff --git a/docs/toolchains.md b/docs/toolchains.md index 09aaed412b..80884baedf 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -242,6 +242,9 @@ existing attributes: {attr}`python.single_version_platform_override.coverage_tool`. * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. +* Adding additional Python versions dynamically from a manifest file or URL + via {attr}`python.override.add_runtime_manifest_files` or + {attr}`python.override.add_runtime_manifest_urls`. ### Registering custom runtimes @@ -310,6 +313,73 @@ Added support for custom platform names, `target_compatible_with`, and `target_settings` with `single_version_platform_override`. ::: +### Registering runtimes from a manifest + +If you want to register multiple custom runtimes or versions at once, you can +use a python-build-standalone manifest file. This is useful if you want to +adopt new versions that are not yet built into `rules_python` without having +to manually define each one using `single_version_platform_override`. + +To do this, specify the `add_runtime_manifest_files` or +`add_runtime_manifest_urls` (and `runtime_manifest_sha`) attributes in +`python.override` in your `MODULE.bazel`. + +In the example below, we register all runtimes available in a specific local +or remote PBS release manifest: + +```starlark +# File: MODULE.bazel +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.override( + add_runtime_manifest_files = [ + "@//:SHA256SUMS", + ], + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + base_url = "https://example.com/downloads", + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", +) +``` + +#### Manifest file format + +The manifest must be a plain text file where each line contains the SHA256 hash +and the location of a runtime archive, separated by whitespace: + +``` + +``` + +The `` can be either: +- A relative filename (e.g., + `cpython-3.10.20+20260414-x86_64-unknown-linux-gnu-install_only.tar.zst`). + In this case, the download URL is constructed by appending the filename to + the `base_url` attribute (if using `add_runtime_manifest_files`) or to the + parent directory of each URL in `add_runtime_manifest_urls` (treating them + as mirrors). +- An absolute URL (e.g., + `https://example.com/downloads/cpython-3.10.20+20260414-x86_64-unknown-linux-gnu-install_only.tar.zst`). + In this case, the URL is used directly to download the archive. + +In both cases, the filename or the last path segment of the URL must follow +the standard python-build-standalone naming convention. `rules_python` parses +this name to extract runtime metadata (such as Python version, target +architecture, operating system, and libc). + +Notes: +- `rules_python` will read or download the manifest, parse it, and + automatically register toolchains for all valid Python runtimes found in it + that match supported platforms. +- Only runtimes matching known platforms in `rules_python` will be registered. + +:::{versionadded} VERSION_NEXT_FEATURE +Added support for registering runtimes from a manifest using +`add_runtime_manifest_files`, `add_runtime_manifest_urls`, and +`runtime_manifest_sha` in `python.override`. +::: + + ### Using defined toolchains from WORKSPACE It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example, diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 6ab3d546be..b54c198069 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -252,6 +252,11 @@ bzl_library( srcs = ["normalize_name.bzl"], ) +bzl_library( + name = "pbs_manifest_bzl", + srcs = ["pbs_manifest.bzl"], +) + bzl_library( name = "precompile_bzl", srcs = ["precompile.bzl"], @@ -274,6 +279,7 @@ bzl_library( srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":pbs_manifest_bzl", ":platform_info_bzl", ":python_register_toolchains_bzl", ":pythons_hub_bzl", diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl new file mode 100644 index 0000000000..e343a802b3 --- /dev/null +++ b/python/private/pbs_manifest.bzl @@ -0,0 +1,153 @@ +"""Helper functions to parse python-build-standalone manifests.""" + +def parse_filename(filename): + """Parses a python-build-standalone filename (or URL) into its components. + + See https://gregoryszorc.com/docs/python-build-standalone/main/running.html + + Example: cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst + + Args: + filename: The filename or URL of the python-build-standalone release asset. + + Returns: + A dictionary of parsed components if parsed successfully, else None. + """ + basename = filename.rpartition("/")[-1] + if basename.endswith(".tar.zst"): + name = basename.removesuffix(".tar.zst") + elif basename.endswith(".tar.gz"): + name = basename.removesuffix(".tar.gz") + else: + return None + + if not name.startswith("cpython-"): + return None + name = name.removeprefix("cpython-") + + left, plus, tail = name.partition("+") + if plus: + python_version = left + build_version, sep, rest = tail.partition("-") + if not sep: + return None + else: + python_version, sep, rest = left.partition("-") + if not sep: + return None + build_version = "" + + arch, sep, rest = rest.partition("-") + if not sep: + return None + + microarch = "" + arch_base, sep_v, microarch_num = arch.partition("_v") + if sep_v: + arch = arch_base + microarch = "v" + microarch_num + + vendor, sep, rest = rest.partition("-") + if not sep: + return None + + os, sep, rest = rest.partition("-") + if not sep: + return None + + libc = "" + next_part, _, remaining = rest.partition("-") + if os == "linux" and next_part in ["gnu", "musl"]: + libc = next_part + flavor = remaining + elif os == "windows" and next_part == "msvc": + libc = next_part + flavor = remaining + else: + libc = "" + flavor = rest + + freethreaded = False + if flavor.startswith("freethreaded+"): + freethreaded = True + flavor = flavor.removeprefix("freethreaded+") + elif flavor.startswith("freethreaded-"): + freethreaded = True + flavor = flavor.removeprefix("freethreaded-") + elif flavor == "freethreaded": + freethreaded = True + flavor = "" + + archive_flavor = "" + if flavor.endswith("-full"): + archive_flavor = "full" + flavor = flavor.removesuffix("-full") + elif flavor == "full": + archive_flavor = "full" + flavor = "" + elif flavor.endswith("-install_only_stripped"): + archive_flavor = "install_only_stripped" + flavor = flavor.removesuffix("-install_only_stripped") + elif flavor == "install_only_stripped": + archive_flavor = "install_only_stripped" + flavor = "" + elif flavor.endswith("-install_only"): + archive_flavor = "install_only" + flavor = flavor.removesuffix("-install_only") + elif flavor == "install_only": + archive_flavor = "install_only" + flavor = "" + + return { + "arch": arch, + "archive_flavor": archive_flavor, + "build_version": build_version, + "flavor": flavor, + "freethreaded": freethreaded, + "libc": libc, + "location": filename, + "microarch": microarch, + "os": os, + "python_version": python_version, + "vendor": vendor, + } + +def parse_sha_manifest(content): + """Parses the SHA256SUMS file content into a list of structs. + + Args: + content: The raw content of the manifest file. + + Returns: + A list of structs capturing the parsed components of each valid entry. + Each struct contains the following fields: + - arch: CPU architecture (e.g., "x86_64"). + - archive_flavor: Release asset archive type (e.g., "full", "install_only"). + - build_version: Standalone release date (e.g., "20260414"). + - location: Full package filename or URL (e.g., "cpython-3.11.15..." or "https://..."). + - flavor: Build configuration flavor (e.g., "install_only"). + - freethreaded: Whether the build is free-threaded (boolean). + - libc: C library type (e.g., "gnu", "musl", "msvc", or ""). + - microarch: Microarchitecture level (e.g., "v2", "v3", or ""). + - os: Operating system (e.g., "linux", "darwin", "windows"). + - python_version: Python semver version (e.g., "3.11.15"). + - sha256: SHA256 integrity hash of the release asset. + - vendor: Platform vendor (e.g., "unknown", "apple"). + """ + results = [] + for line in content.split("\n"): + line = line.strip() + if not line: + continue + parts = [p for p in line.split(" ") if p] + if len(parts) != 2: + continue + sha256, filename = parts + + parsed = parse_filename(filename) + if parsed: + results.append(struct( + sha256 = sha256, + **parsed + )) + return results diff --git a/python/private/python.bzl b/python/private/python.bzl index 6abc81e3d2..73b2d19839 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -18,6 +18,7 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") +load(":pbs_manifest.bzl", "parse_sha_manifest") load(":platform_info.bzl", "platform_info") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") @@ -76,7 +77,7 @@ def parse_modules(*, module_ctx, logger = None, _fail = fail): # Map of string Major.Minor or Major.Minor.Patch to the toolchain_info struct global_toolchain_versions = {} - config = _get_toolchain_config(modules = module_ctx.modules, _fail = _fail) + config = _get_toolchain_config(mctx = module_ctx, modules = module_ctx.modules, _fail = _fail) default_python_version = _compute_default_python_version(module_ctx) @@ -741,10 +742,89 @@ def _override_defaults(*overrides, modules, _fail = fail, default): override.fn(tag = tag, _fail = _fail, default = default) -def _get_toolchain_config(*, modules, _fail = fail): +def _populate_from_pbs_manifest( + *, + mctx, + add_runtime_manifest_urls = [], + add_runtime_manifest_files = [], + runtime_manifest_sha = "", + base_url = "", + available_versions, + _fail): + manifest_contents = [] + + if add_runtime_manifest_urls: + manifest_path = mctx.path("runtime_manifest") + result = mctx.download( + url = add_runtime_manifest_urls, + output = manifest_path, + sha256 = runtime_manifest_sha, + ) + if not result.success: + _fail("Failed to download manifest from {}: {}".format(add_runtime_manifest_urls, result)) + return + manifest_contents.append(mctx.read(manifest_path)) + + for manifest_file in add_runtime_manifest_files: + manifest_contents.append(mctx.read(manifest_file, watch = "yes")) + + if not manifest_contents: + return + + base_download_urls = [url.rpartition("/")[0] for url in add_runtime_manifest_urls] + if not base_download_urls and base_url: + base_download_urls = [base_url] + + entries = [] + for content in manifest_contents: + entries.extend(parse_sha_manifest(content)) + + # We don't model archive_flavor via flags yet, so have to pick one. + # Preference is given to install_only because its smaller + entries = sorted( + entries, + key = lambda e: {"full": 3, "install_only": 1, "install_only_stripped": 2}.get(e.archive_flavor, 4), + ) + + for entry in entries: + location = entry.location + sha256 = entry.sha256 + py_version = entry.python_version + + # Fallback to matching against PLATFORMS keys as before to ensure compatibility + # with rules_python expected platform keys. + matched_platform = None + for platform in PLATFORMS.keys(): + if platform in location: + matched_platform = platform + break + + if not matched_platform: + continue + + if entry.archive_flavor not in ["install_only", "install_only_stripped", "full"]: + continue + + v_dict = available_versions.setdefault(py_version, {}) + if matched_platform in v_dict.get("sha256", {}): + continue + + if "://" in location: + urls = [location] + else: + urls = ["{}/{}".format(b_url, location) for b_url in base_download_urls] + + strip_prefix = "python/install" if entry.archive_flavor == "full" else "python" + + v_dict.setdefault("sha256", {})[matched_platform] = sha256 + v_dict.setdefault("url", {})[matched_platform] = urls + v_dict.setdefault("strip_prefix", {})[matched_platform] = strip_prefix + +def _get_toolchain_config(*, mctx, modules, _fail = fail): """Computes the configs for toolchains. Args: + mctx: The module context. modules: The modules from module_ctx _fail: Function to call for failing; only used for testing. @@ -786,6 +866,21 @@ def _get_toolchain_config(*, modules, _fail = fail): else: available_versions[py_version]["url"] = dict(url) + # Check for add_runtime_manifest_urls or add_runtime_manifest_files in override tags in root module + root_module = modules[0] if modules else None + if root_module and root_module.is_root: + for tag in root_module.tags.override: + if tag.add_runtime_manifest_urls or tag.add_runtime_manifest_files: + _populate_from_pbs_manifest( + mctx = mctx, + add_runtime_manifest_urls = tag.add_runtime_manifest_urls, + add_runtime_manifest_files = tag.add_runtime_manifest_files, + runtime_manifest_sha = tag.runtime_manifest_sha, + base_url = tag.base_url, + available_versions = available_versions, + _fail = _fail, + ) + default = { "base_url": DEFAULT_RELEASE_BASE_URL, "platforms": dict(PLATFORMS), # Copy so it's mutable. @@ -1111,6 +1206,34 @@ _override = tag_class( ::: """, attrs = { + "add_runtime_manifest_files": attr.label_list( + mandatory = False, + allow_files = True, + doc = """ +Labels pointing to local python-build-standalone manifest files (e.g., `SHA256SUMS`). + +Example: +`//my/custom/manifest:SHA256SUMS` + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "add_runtime_manifest_urls": attr.string_list( + mandatory = False, + doc = """ +URLs pointing to python-build-standalone manifest files (e.g., SHA256SUMS). + +Example: +`https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS` + +Note that `/latest/` can be used in place of a specific release date (e.g., `20260414`) to automatically use the latest release: +`https://github.com/astral-sh/python-build-standalone/releases/latest/download/SHA256SUMS` + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), "add_target_settings": attr.string_list( mandatory = False, doc = """\ @@ -1180,6 +1303,15 @@ The values in this mapping override the default values and do not replace them. default = {}, ), "register_all_versions": attr.bool(default = False, doc = "Add all versions"), + "runtime_manifest_sha": attr.string( + mandatory = False, + doc = """ +SHA256 hash for the add_runtime_manifest_urls. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), } | AUTH_ATTRS, ) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index a6027fc3d4..904fb4c247 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -104,6 +104,10 @@ rules_python_integration_test( workspace_path = "py_cc_toolchain_registered", ) +rules_python_integration_test( + name = "runtime_manifests_test", +) + rules_python_integration_test( name = "custom_commands_test", py_main = "custom_commands_test.py", diff --git a/tests/integration/runtime_manifests/.bazelrc b/tests/integration/runtime_manifests/.bazelrc new file mode 100644 index 0000000000..d4d45a5ea7 --- /dev/null +++ b/tests/integration/runtime_manifests/.bazelrc @@ -0,0 +1,4 @@ +# Copy of fast-tests config +common:fast-tests --build_tests_only=true +common:fast-tests --build_tag_filters=-large,-enormous,-integration-test +common:fast-tests --test_tag_filters=-large,-enormous,-integration-test diff --git a/tests/integration/runtime_manifests/BUILD.bazel b/tests/integration/runtime_manifests/BUILD.bazel new file mode 100644 index 0000000000..4746fd419b --- /dev/null +++ b/tests/integration/runtime_manifests/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_python//python:py_test.bzl", "py_test") + +py_test( + name = "basic_test", + srcs = ["basic_test.py"], + python_version = "3.11", +) diff --git a/tests/integration/runtime_manifests/MODULE.bazel b/tests/integration/runtime_manifests/MODULE.bazel new file mode 100644 index 0000000000..6891b07818 --- /dev/null +++ b/tests/integration/runtime_manifests/MODULE.bazel @@ -0,0 +1,19 @@ +module(name = "runtime_manifests") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + register_all_versions = True, + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", +) +python.toolchain( + python_version = "3.11.15", +) diff --git a/tests/integration/runtime_manifests/WORKSPACE b/tests/integration/runtime_manifests/WORKSPACE new file mode 100644 index 0000000000..8277ec8090 --- /dev/null +++ b/tests/integration/runtime_manifests/WORKSPACE @@ -0,0 +1 @@ +# Workspace boundary file required by rules_bazel_integration_test diff --git a/tests/integration/runtime_manifests/basic_test.py b/tests/integration/runtime_manifests/basic_test.py new file mode 100644 index 0000000000..35f0e93b6e --- /dev/null +++ b/tests/integration/runtime_manifests/basic_test.py @@ -0,0 +1,27 @@ +import datetime +import platform +import sys +import unittest + + +class BasicTest(unittest.TestCase): + def test_basic(self): + print("Hello World from Python {}!".format(sys.version)) + print("Interpreter executable path: {}".format(sys.executable)) + + # Verify that the hermetic interpreter inside Bazel's output/sandbox tree is used + self.assertIn(".cache/bazel", sys.executable) + + # Verify that the exact custom version (3.11.15) parsed from the manifest is used + self.assertEqual(sys.version_info[:3], (3, 11, 15)) + + # Verify that the exact build version (20260414) parsed from the manifest is used + buildno, builddate = platform.python_build() + date_str = " ".join(builddate.split()[:3]) + dt = datetime.datetime.strptime(date_str, "%b %d %Y") + formatted_date = dt.strftime("%Y%m%d") + self.assertEqual(formatted_date, "20260414") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/BUILD.bazel b/tests/python/BUILD.bazel index 2553536b63..887fe969b5 100644 --- a/tests/python/BUILD.bazel +++ b/tests/python/BUILD.bazel @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":python_tests.bzl", "python_test_suite") +load(":python_tests.bzl", "register_python_tests") -python_test_suite(name = "python_tests") +register_python_tests(name = "python_tests") diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 5db74265be..cd7383942c 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -16,6 +16,7 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING") load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") @@ -878,3 +879,17 @@ def python_test_suite(name): name: the name of the test suite """ test_suite(name = name, basic_tests = _tests) + +def register_python_tests(name): + """Registers the python tests if Bzlmod is enabled, otherwise defines an empty test_suite. + + Args: + name: The name of the test target. + """ + if BZLMOD_ENABLED: + python_test_suite(name = name) + else: + native.test_suite( + name = name, + tests = [], + ) diff --git a/tests/python_bzlmod_ext/BUILD.bazel b/tests/python_bzlmod_ext/BUILD.bazel new file mode 100644 index 0000000000..0add4e7690 --- /dev/null +++ b/tests/python_bzlmod_ext/BUILD.bazel @@ -0,0 +1,7 @@ +load(":test_helpers.bzl", "register_python_bzlmod_ext_tests") + +register_python_bzlmod_ext_tests( + name = "python_bzlmod_ext_tests", + parse_sha_manifest_name = "parse_sha_manifest_tests", + runtime_manifests_name = "runtime_manifests_tests", +) diff --git a/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl new file mode 100644 index 0000000000..4ddcdcc46e --- /dev/null +++ b/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl @@ -0,0 +1,183 @@ +"""Tests for manifest parsing Starlark functions.""" + +load("@bazel_skylib//lib:structs.bzl", "structs") +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:pbs_manifest.bzl", "parse_filename", "parse_sha_manifest") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_parse_filename_baseline(name): + """Sets up the baseline filename parsing test. + + Args: + name: The name of the test. + """ + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_parse_filename_baseline_impl, + ) + +def _test_parse_filename_baseline_impl(env, target): + _ = target # @unused + + # 1. Baseline + parsed1 = parse_filename("cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz") + env.expect.that_dict(parsed1).contains_exactly({ + "arch": "x86_64", + "archive_flavor": "install_only", + "build_version": "20260414", + "flavor": "", + "freethreaded": False, + "libc": "gnu", + "location": "cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "vendor": "unknown", + }) + + # 2. Microarch + parsed2 = parse_filename("cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst") + env.expect.that_dict(parsed2).contains_exactly({ + "arch": "x86_64", + "archive_flavor": "full", + "build_version": "20260414", + "flavor": "lto", + "freethreaded": False, + "libc": "musl", + "location": "cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst", + "microarch": "v2", + "os": "linux", + "python_version": "3.10.20", + "vendor": "unknown", + }) + + # 3. Freethreaded + parsed3 = parse_filename("cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst") + env.expect.that_dict(parsed3).contains_exactly({ + "arch": "aarch64", + "archive_flavor": "full", + "build_version": "20260414", + "flavor": "pgo+lto", + "freethreaded": True, + "libc": "", + "location": "cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst", + "microarch": "", + "os": "darwin", + "python_version": "3.13.13", + "vendor": "apple", + }) + + # 4. Invalid + parsed4 = parse_filename("invalid-filename.tar.gz") + env.expect.that_bool(parsed4 == None).equals(True) + + # 5. Full URL (should return the original URL as location) + parsed5 = parse_filename("https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz") + env.expect.that_dict(parsed5).contains_exactly({ + "arch": "x86_64", + "archive_flavor": "install_only", + "build_version": "20260414", + "flavor": "", + "freethreaded": False, + "libc": "gnu", + "location": "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "vendor": "unknown", + }) + +_tests.append(_test_parse_filename_baseline) + +def _test_parse_sha_manifest(name): + """Sets up the manifest file parsing test. + + Args: + name: The name of the test. + """ + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_parse_sha_manifest_impl, + ) + +def _test_parse_sha_manifest_impl(env, target): + _ = target # @unused + content = """ +8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz +a57ffd435652092d16b30e783f9826c55e9c64b0f0a72cbae0a9f39e663137fb cpython-3.11.15+20260414-aarch64-apple-darwin-install_only.tar.gz +ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f https://example.com/cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst +1111111111111111111111111111111111111111111111111111111111111111 cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst +""" + parsed = parse_sha_manifest(content) + env.expect.that_collection(parsed).has_size(4) + + env.expect.that_dict(structs.to_dict(parsed[0])).contains_exactly({ + "arch": "x86_64", + "archive_flavor": "install_only", + "build_version": "20260414", + "flavor": "", + "freethreaded": False, + "libc": "gnu", + "location": "cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "sha256": "8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc", + "vendor": "unknown", + }) + + env.expect.that_dict(structs.to_dict(parsed[2])).contains_exactly({ + "arch": "x86_64", + "archive_flavor": "full", + "build_version": "20260414", + "flavor": "lto", + "freethreaded": False, + "libc": "musl", + "location": "https://example.com/cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst", + "microarch": "v2", + "os": "linux", + "python_version": "3.10.20", + "sha256": "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", + "vendor": "unknown", + }) + + env.expect.that_dict(structs.to_dict(parsed[3])).contains_exactly({ + "arch": "aarch64", + "archive_flavor": "full", + "build_version": "20260414", + "flavor": "pgo+lto", + "freethreaded": True, + "libc": "", + "location": "cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst", + "microarch": "", + "os": "darwin", + "python_version": "3.13.13", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "vendor": "apple", + }) + +_tests.append(_test_parse_sha_manifest) + +def parse_sha_manifest_test_suite(name): + """Defines the test suite for manifest parsing. + + Args: + name: The name of the test suite. + """ + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/python_bzlmod_ext/runtime_manifests_tests.bzl b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl new file mode 100644 index 0000000000..c46cab1958 --- /dev/null +++ b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl @@ -0,0 +1,163 @@ +"""Starlark unit tests for dynamic toolchain registration via manifests.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:python_ext.bzl", "python_ext") # buildifier: disable=bzl-visibility + +_tests = [] + +_mock_logger = repo_utils.logger( + name = "mock", + verbosity_level = "ERROR", +) + +def _test_dynamic_manifest_toolchains(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_dynamic_manifest_toolchains_impl, + ) + +def _test_dynamic_manifest_toolchains_impl(env, target): + _ = target # @unused + + # Construct Bzlmod mock module locally inside the test execution block. + # We test using virtual patch version "3.11.99" (not present in TOOL_VERSIONS) + # so that the populated config contains ONLY our dynamically parsed manifest keys + # without any pre-populated multi-platform templates, allowing exact dictionary match! + root_module = python_ext.module( + name = "runtime_manifests", + override = [ + python_ext.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", + register_all_versions = True, + ), + ], + defaults = [ + python_ext.defaults( + python_version = "3.11.99", + ), + ], + ) + + # Pre-populate mock_files directly to bypass download output struct key mismatch in mock read lookups. + mock_mctx = mocks.mctx( + modules = [root_module], + mock_files = { + "runtime_manifest": """ +01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst +8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz +""", + }, + ) + + res = parse_modules( + module_ctx = mock_mctx, + logger = _mock_logger, + ) + + tool_versions = res.config.default["tool_versions"] + env.expect.that_bool("3.11.99" in tool_versions).equals(True) + + version_info = tool_versions["3.11.99"] + + # Assert on the entire dictionary at once! + env.expect.that_dict(version_info).contains_exactly({ + "sha256": { + "x86_64-unknown-linux-gnu": "8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc", + }, + "strip_prefix": { + "x86_64-unknown-linux-gnu": "python", + }, + "url": { + "x86_64-unknown-linux-gnu": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + ], + }, + }) + +_tests.append(_test_dynamic_manifest_toolchains) + +def _test_dynamic_manifest_files(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_dynamic_manifest_files_impl, + ) + +def _test_dynamic_manifest_files_impl(env, target): + _ = target # @unused + + root_module = python_ext.module( + name = "runtime_manifests", + override = [ + python_ext.override( + add_runtime_manifest_files = [ + Label("//:SHA256SUMS"), + ], + base_url = "https://example.com/dl", + register_all_versions = True, + ), + ], + defaults = [ + python_ext.defaults( + python_version = "3.12.99", + ), + ], + ) + + mock_mctx = mocks.mctx( + modules = [root_module], + mock_files = { + str(Label("//:SHA256SUMS")): """ +01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a cpython-3.12.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst +""", + }, + ) + + res = parse_modules( + module_ctx = mock_mctx, + logger = _mock_logger, + ) + + tool_versions = res.config.default["tool_versions"] + env.expect.that_bool("3.12.99" in tool_versions).equals(True) + + version_info = tool_versions["3.12.99"] + + env.expect.that_dict(version_info).contains_exactly({ + "sha256": { + "x86_64-unknown-linux-gnu": "01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a", + }, + "strip_prefix": { + "x86_64-unknown-linux-gnu": "python/install", + }, + "url": { + "x86_64-unknown-linux-gnu": [ + "https://example.com/dl/cpython-3.12.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst", + ], + }, + }) + +_tests.append(_test_dynamic_manifest_files) + +def runtime_manifests_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/python_bzlmod_ext/test_helpers.bzl b/tests/python_bzlmod_ext/test_helpers.bzl new file mode 100644 index 0000000000..f6c177650a --- /dev/null +++ b/tests/python_bzlmod_ext/test_helpers.bzl @@ -0,0 +1,48 @@ +# Copyright 2026 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers to conditionally register tests depending on Bzlmod enablement.""" + +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load(":parse_sha_manifest_tests.bzl", "parse_sha_manifest_test_suite") +load(":runtime_manifests_tests.bzl", "runtime_manifests_test_suite") + +def register_python_bzlmod_ext_tests(name, parse_sha_manifest_name, runtime_manifests_name): + """Registers the Bzlmod extension tests if Bzlmod is enabled, otherwise defines empty test_suites. + + Args: + name: The name of the master test_suite target. + parse_sha_manifest_name: The name of the parse_sha_manifest test target. + runtime_manifests_name: The name of the runtime_manifests test target. + """ + if BZLMOD_ENABLED: + parse_sha_manifest_test_suite(name = parse_sha_manifest_name) + runtime_manifests_test_suite(name = runtime_manifests_name) + else: + native.test_suite( + name = parse_sha_manifest_name, + tests = [], + ) + native.test_suite( + name = runtime_manifests_name, + tests = [], + ) + + native.test_suite( + name = name, + tests = [ + parse_sha_manifest_name, + runtime_manifests_name, + ], + ) diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl index f20a6c7263..f7b5b0ee02 100644 --- a/tests/support/mocks/python_ext.bzl +++ b/tests/support/mocks/python_ext.bzl @@ -26,6 +26,7 @@ def _module(name = "rules_python", is_root = True, **tags): def _override(**kwargs): """Creates a mock python.override tag with default values.""" attrs = { + "add_runtime_manifest_files": [], "add_runtime_manifest_urls": [], "add_target_settings": [], "available_python_versions": [], diff --git a/tools/private/debug/print_defined_toolchains.sh b/tools/private/debug/print_defined_toolchains.sh new file mode 100755 index 0000000000..1694dc975c --- /dev/null +++ b/tools/private/debug/print_defined_toolchains.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Programmatically probe which repository target name is resolved successfully inside this workspace +if bazel query @pythons_hub//... >/dev/null 2>&1; then + HUB_REPO="@pythons_hub" +elif bazel query @@rules_python++python+pythons_hub//... >/dev/null 2>&1; then + HUB_REPO="@@rules_python++python+pythons_hub" +else + HUB_REPO="@@+python+pythons_hub" +fi + +# Query standard toolchains inside the resolved hub repository, excluding CC and Exec Tools toolchains. +bazel query "kind('toolchain', ${HUB_REPO}//...) - filter('_py_cc_toolchain$', ${HUB_REPO}//...) - filter('_py_exec_tools_toolchain$', ${HUB_REPO}//...)" "$@"