From 81d06b99e098ead24d4c935bd538c36de0909226 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Mon, 3 Aug 2026 19:15:37 +0000 Subject: [PATCH 1/8] fix(serve): propagate additional_model_data_sources for JumpStart models ModelBuilder._build_for_jumpstart() copied image_uri, env, enable_network_isolation, model_reference_arn, and model_data from the resolved JumpStart init_kwargs, but never propagated additional_model_data_sources. Any JumpStart model whose spec declares an extra S3 data channel (e.g. an EAGLE speculative decoding draft model, a LoRA adapter, or a tokenizer override) therefore had that channel silently dropped from the CreateModel call, and the container failed at runtime trying to read artifacts from a path that was never mounted. Propagate additional_model_data_sources from init_kwargs onto the builder so _prepare_container_def_base attaches it to the CreateModel request. Each source is filtered to only the keys the CreateModel API accepts (ChannelName, S3DataSource); JumpStart specs may include extra metadata keys such as HostingEulaKey that the API rejects. Adds unit tests for both the propagation+filtering path and the no-op case when the spec declares no additional data sources. --- .../sagemaker/serve/model_builder_servers.py | 18 +++++ .../servers/test_model_builder_servers.py | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index 08a77e0d1e..f39aa51bba 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1014,6 +1014,24 @@ def _build_for_jumpstart(self) -> Model: if getattr(init_kwargs, "model_reference_arn", None): self.model_reference_arn = init_kwargs.model_reference_arn + # Propagate additional model data sources from the JumpStart spec (e.g. + # speculative decoding draft models like EAGLE, LoRA adapters, or any + # extra S3 data channel). Without this, sources declared in the spec are + # silently dropped from the CreateModel call and the container fails to + # find the referenced artifacts at runtime. + additional_model_data_sources = getattr( + init_kwargs, "additional_model_data_sources", None + ) + if isinstance(additional_model_data_sources, list) and additional_model_data_sources: + # Filter each source to only the keys the CreateModel API accepts. + # JumpStart specs may include extra metadata keys (e.g. HostingEulaKey) + # that the API rejects. + _api_keys = {"ChannelName", "S3DataSource"} + self.additional_model_data_sources = [ + {key: value for key, value in source.items() if key in _api_keys} + for source in additional_model_data_sources + ] + # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: if ( diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 4fc1a558ba..c9d284e860 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1004,6 +1004,75 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init mock_create.assert_called_once() + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_propagates_additional_model_data_sources( + self, mock_create, mock_prepare, mock_init + ): + """additional_model_data_sources from the JumpStart spec are propagated + to the builder and filtered to only API-accepted keys.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + # JumpStart spec includes a metadata key (HostingEulaKey) that the + # CreateModel API does not accept; it must be filtered out. + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + } + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_without_additional_model_data_sources( + self, mock_create, mock_prepare, mock_init + ): + """Models whose spec has no additional_model_data_sources are unaffected.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = None + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + + self.builder._build_for_jumpstart() + + self.assertIsNone( + getattr(self.builder, "additional_model_data_sources", None) + ) + class TestDeployWrappers(unittest.TestCase): """Test deploy wrapper methods.""" From bc112ff22c345fc0686847f7376d04b073329e5e Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 13:19:48 +0000 Subject: [PATCH 2/8] fix(serve): enforce accept_eula on gated additional model data sources A source carrying a truthy HostingEulaKey is gated and requires license acceptance, exactly like the base model. container_def folds accept_eula into the base model's ModelAccessConfig but passes additional sources through untouched, so the acceptance is translated onto each gated source here: fold ModelAccessConfig={AcceptEula: True} into its S3DataSource when accept_eula is set, raise the same error the base model raises when it is not. Ungated sources just drop the JumpStart-internal HostingEulaKey the CreateModel API rejects. Test coverage mirrors the v2 helper's case inventory: multiple public sources, ungated ignores acceptance, mixed public+gated, gated accept, gated missing/false acceptance raises, plus a raw-spec-JSON end-to-end test driving the real JumpStartModelSpecs parsing and factory shaping. --- .../sagemaker/serve/model_builder_servers.py | 47 ++- .../servers/test_model_builder_servers.py | 392 +++++++++++++++++- 2 files changed, 423 insertions(+), 16 deletions(-) diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index f39aa51bba..57f43d7504 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1014,23 +1014,44 @@ def _build_for_jumpstart(self) -> Model: if getattr(init_kwargs, "model_reference_arn", None): self.model_reference_arn = init_kwargs.model_reference_arn - # Propagate additional model data sources from the JumpStart spec (e.g. - # speculative decoding draft models like EAGLE, LoRA adapters, or any - # extra S3 data channel). Without this, sources declared in the spec are - # silently dropped from the CreateModel call and the container fails to - # find the referenced artifacts at runtime. + # Propagate additional model data sources resolved from the JumpStart + # spec (e.g. speculative decoding draft models like EAGLE). The JumpStart + # factory (_add_additional_model_data_sources_to_kwargs) already returns + # these in CreateModel API shape via camel_case_to_pascal_case(...). + # + # A source carrying a truthy ``HostingEulaKey`` is gated and requires the + # user to accept its end-user license agreement, exactly like the base + # model. ``container_def`` folds ``accept_eula`` into the base model's + # ModelAccessConfig but passes additional sources through untouched, so + # the acceptance must be translated onto each gated source here: enforce + # ``accept_eula`` and move it into the source's ``ModelAccessConfig`` + # (mirroring _add_model_access_configs_to_model_data_sources and the base + # model's handling in _create_model). Ungated sources simply drop the + # JumpStart-internal ``HostingEulaKey`` metadata, which the CreateModel + # API rejects. Without any of this, sources declared in the spec are + # dropped from the CreateModel call and the container fails to find the + # referenced artifacts at runtime. additional_model_data_sources = getattr( init_kwargs, "additional_model_data_sources", None ) if isinstance(additional_model_data_sources, list) and additional_model_data_sources: - # Filter each source to only the keys the CreateModel API accepts. - # JumpStart specs may include extra metadata keys (e.g. HostingEulaKey) - # that the API rejects. - _api_keys = {"ChannelName", "S3DataSource"} - self.additional_model_data_sources = [ - {key: value for key, value in source.items() if key in _api_keys} - for source in additional_model_data_sources - ] + accept_eula = getattr(self, "accept_eula", None) + sanitized_sources = [] + for source in additional_model_data_sources: + sanitized_source = dict(source) + if sanitized_source.pop("HostingEulaKey", None): + if not accept_eula: + raise ValueError( + "accept_eula must be set to True to deploy this model. " + "Please set accept_eula=True on the ModelBuilder instance " + "to confirm you have read and accepted the end-user license " + "agreement for this model." + ) + s3_data_source = dict(sanitized_source.get("S3DataSource", {})) + s3_data_source["ModelAccessConfig"] = {"AcceptEula": True} + sanitized_source["S3DataSource"] = s3_data_source + sanitized_sources.append(sanitized_source) + self.additional_model_data_sources = sanitized_sources # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index c9d284e860..03f8c908c4 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1011,13 +1011,14 @@ def test_build_propagates_additional_model_data_sources( self, mock_create, mock_prepare, mock_init ): """additional_model_data_sources from the JumpStart spec are propagated - to the builder and filtered to only API-accepted keys.""" + to the builder with the JumpStart-internal HostingEulaKey metadata + stripped (mirroring _add_model_access_configs_to_model_data_sources).""" mock_init_kwargs = Mock() mock_init_kwargs.image_uri = "djl-inference:0.21.0" mock_init_kwargs.env = {} mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - # JumpStart spec includes a metadata key (HostingEulaKey) that the - # CreateModel API does not accept; it must be filtered out. + # JumpStart spec includes an internal metadata key (HostingEulaKey) that + # the CreateModel API does not accept; it must be stripped. mock_init_kwargs.additional_model_data_sources = [ { "ChannelName": "draft_model", @@ -1073,6 +1074,391 @@ def test_build_without_additional_model_data_sources( getattr(self.builder, "additional_model_data_sources", None) ) + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_gated_additional_model_data_source_with_accept_eula( + self, mock_create, mock_prepare, mock_init + ): + """A gated additional source (truthy HostingEulaKey) has the acceptance + folded into its ModelAccessConfig when accept_eula is set, mirroring the + base model's EULA handling.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + "ModelAccessConfig": {"AcceptEula": True}, + }, + } + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_gated_additional_model_data_source_without_accept_eula_raises( + self, mock_create, mock_prepare, mock_init + ): + """A gated additional source without accept_eula raises, matching the + base model's EULA enforcement.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = None + + with self.assertRaises(ValueError) as ctx: + self.builder._build_for_jumpstart() + self.assertIn("accept_eula must be set to True", str(ctx.exception)) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_multiple_public_additional_model_data_sources_pass_through( + self, mock_create, mock_prepare, mock_init + ): + """Multiple ungated additional sources are all propagated with only the + HostingEulaKey metadata stripped.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + }, + { + "ChannelName": "scripts", + "S3DataSource": { + "S3Uri": "s3://bucket/scripts/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + }, + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + }, + { + "ChannelName": "scripts", + "S3DataSource": { + "S3Uri": "s3://bucket/scripts/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + }, + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_public_additional_model_data_source_ignores_accept_eula( + self, mock_create, mock_prepare, mock_init + ): + """accept_eula on an ungated additional source is ignored: no + ModelAccessConfig is added (mirrors the v2 helper, which only applies + access configs to gated sources).""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + } + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_mixed_additional_model_data_sources_with_accept_eula( + self, mock_create, mock_prepare, mock_init + ): + """With a mix of ungated and gated sources and accept_eula=True, the + ungated source passes through untouched while the gated source gets + ModelAccessConfig folded into its S3DataSource.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + }, + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + }, + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + }, + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + "ModelAccessConfig": {"AcceptEula": True}, + }, + }, + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_gated_additional_model_data_source_accept_eula_false_raises( + self, mock_create, mock_prepare, mock_init + ): + """An explicit accept_eula=False on a gated additional source raises, + matching the v2 helper's rejection of a false acceptance.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = False + + with self.assertRaises(ValueError) as ctx: + self.builder._build_for_jumpstart() + self.assertIn("accept_eula must be set to True", str(ctx.exception)) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_additional_model_data_sources_from_raw_spec_json( + self, mock_create, mock_prepare, mock_init + ): + """End-to-end from a raw JumpStart spec JSON: the real + JumpStartModelSpecs parsing and the real factory shaping helper + (_add_additional_model_data_sources_to_kwargs) produce the API-shape + init kwargs, and _build_for_jumpstart translates them into the final + CreateModel shape (HostingEulaKey stripped, EULA acceptance folded + into the gated source's ModelAccessConfig).""" + from sagemaker.core.jumpstart.factory.utils import ( + _add_additional_model_data_sources_to_kwargs, + ) + from sagemaker.core.jumpstart.types import JumpStartModelSpecs + + # Raw spec JSON in the snake_case shape published to the JumpStart + # content bucket (hosting_additional_data_sources.speculative_decoding). + raw_spec_json = { + "model_id": "pytorch-ic-mobilenet-v2", + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/draft/model/", + }, + "provider": {"name": "JumpStart", "classification": "ungated"}, + }, + { + "channel_name": "gated_draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/gated/draft/", + }, + "hosting_eula_key": "fmhMetadata/eula/llama3Eula.txt", + "provider": {"name": "JumpStart", "classification": "gated"}, + }, + ] + }, + } + + # Run the real factory shaping over the real parsed spec, exactly as + # get_init_kwargs does, to produce the API-shape init kwargs. + mock_init_kwargs = Mock() + mock_init_kwargs.specs = JumpStartModelSpecs(raw_spec_json) + mock_init_kwargs.region = "us-west-2" + mock_init_kwargs.additional_model_data_sources = None + mock_init_kwargs = _add_additional_model_data_sources_to_kwargs( + kwargs=mock_init_kwargs + ) + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init.return_value = mock_init_kwargs + + # The factory shaping keeps HostingEulaKey (internal metadata) in the + # API-shape kwargs; that is what _build_for_jumpstart must translate. + self.assertEqual( + [s.get("HostingEulaKey") for s in mock_init_kwargs.additional_model_data_sources], + [None, "fmhMetadata/eula/llama3Eula.txt"], + ) + + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", + }, + }, + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": ( + "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/" + ), + "ModelAccessConfig": {"AcceptEula": True}, + }, + }, + ], + ) + class TestDeployWrappers(unittest.TestCase): """Test deploy wrapper methods.""" From 46038f949ef09efd2a63be7a86b32b2a30d7a55a Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 14:28:35 +0000 Subject: [PATCH 3/8] test(serve): drive additional-source EULA tests through the user flow Replace the mock-harness additional_model_data_sources tests with user-flow tests that exercise the documented user contract: construct a real ModelBuilder, set accept_eula, and call build(). Only the spec fetch boundary (JumpStartModelsAccessor.get_model_specs / _get_manifest) and the AWS boundary (_prepare_for_mode / _create_model, mock session) are mocked, mirroring how master-v2 tested JumpStartModel. Raw spec JSON fixtures live in data/jumpstart_specs.json as a base spec plus overlay variants (public model without/with public sources, gated sources, mixed), reusable across tests. Everything between the two boundaries runs for real: JumpStart detection, JumpStartModelSpecs parsing, the get_init_kwargs factory shaping, and the EULA translation in _build_for_jumpstart. --- .../unit/servers/data/jumpstart_specs.json | 898 ++++++++++++++++++ .../servers/test_model_builder_servers.py | 579 ++++------- 2 files changed, 1060 insertions(+), 417 deletions(-) create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json new file mode 100644 index 0000000000..dfedcde98e --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json @@ -0,0 +1,898 @@ +{ + "_comment": "Raw JumpStart spec JSON fixtures for user-flow tests. base_spec is the prototypical pytorch-ic-mobilenet-v2 spec (same prototype the pre-v3 test suite used); each variant is an overlay merged over base_spec by the test loader. The shapes mirror what is published to the JumpStart content bucket (snake_case, hosting_additional_data_sources.speculative_decoding).", + "base_spec": { + "model_id": "pytorch-ic-mobilenet-v2", + "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", + "version": "3.0.6", + "min_sdk_version": "2.189.0", + "training_supported": true, + "incremental_training_supported": true, + "hosting_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", + "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", + "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_use_script_uri": false, + "inference_vulnerable": false, + "inference_dependencies": [], + "inference_vulnerabilities": [], + "training_vulnerable": false, + "training_dependencies": [ + "sagemaker_jumpstart_prepack_script_utilities==1.0.0" + ], + "training_vulnerabilities": [], + "deprecated": false, + "hyperparameters": [ + { + "name": "train_only_top_layer", + "type": "text", + "default": "True", + "options": [ + "True", + "False" + ], + "scope": "algorithm" + }, + { + "name": "epochs", + "type": "int", + "default": 5, + "min": 1, + "max": 1000, + "scope": "algorithm" + }, + { + "name": "learning_rate", + "type": "float", + "default": 0.001, + "min": 1e-08, + "max": 1, + "scope": "algorithm" + }, + { + "name": "batch_size", + "type": "int", + "default": 4, + "min": 1, + "max": 1024, + "scope": "algorithm" + }, + { + "name": "reinitialize_top_layer", + "type": "text", + "default": "Auto", + "options": [ + "Auto", + "True", + "False" + ], + "scope": "algorithm" + }, + { + "name": "sagemaker_submit_directory", + "type": "text", + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "scope": "container" + }, + { + "name": "sagemaker_program", + "type": "text", + "default": "transfer_learning.py", + "scope": "container" + }, + { + "name": "sagemaker_container_log_level", + "type": "text", + "default": "20", + "scope": "container" + } + ], + "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", + "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.1.0", + "training_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", + "inference_environment_variables": [ + { + "name": "SAGEMAKER_PROGRAM", + "type": "text", + "default": "inference.py", + "scope": "container", + "required_for_model_class": true + }, + { + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "type": "text", + "default": "/opt/ml/model/code", + "scope": "container", + "required_for_model_class": false + }, + { + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "type": "text", + "default": "20", + "scope": "container", + "required_for_model_class": false + }, + { + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "type": "text", + "default": "3600", + "scope": "container", + "required_for_model_class": false + }, + { + "name": "ENDPOINT_SERVER_TIMEOUT", + "type": "int", + "default": 3600, + "scope": "container", + "required_for_model_class": true + }, + { + "name": "MODEL_CACHE_ROOT", + "type": "text", + "default": "/opt/ml/model", + "scope": "container", + "required_for_model_class": true + }, + { + "name": "SAGEMAKER_ENV", + "type": "text", + "default": "1", + "scope": "container", + "required_for_model_class": true + }, + { + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "type": "int", + "default": 1, + "scope": "container", + "required_for_model_class": true + } + ], + "metrics": [ + { + "Name": "pytorch-ic:val-accuracy", + "Regex": "val_accuracy: ([0-9\\.]+)" + } + ], + "default_inference_instance_type": "ml.m5.large", + "supported_inference_instance_types": [ + "ml.m5.large", + "ml.m5.xlarge", + "ml.c5.xlarge", + "ml.c5.2xlarge", + "ml.m4.large", + "ml.m4.xlarge" + ], + "default_training_instance_type": "ml.m5.xlarge", + "supported_training_instance_types": [ + "ml.m5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "model_kwargs": {}, + "deploy_kwargs": {}, + "estimator_kwargs": { + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fit_kwargs": {}, + "predictor_specs": { + "supported_content_types": [ + "application/x-image" + ], + "supported_accept_types": [ + "application/json;verbose", + "application/json" + ], + "default_content_type": "application/x-image", + "default_accept_type": "application/json" + }, + "inference_enable_network_isolation": true, + "training_enable_network_isolation": true, + "default_training_dataset_key": "training-datasets/tf_flowers/", + "validation_supported": false, + "fine_tuning_supported": true, + "resource_name_base": "pt-ic-mobilenet-v2", + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-2": { + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-3": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "il-central-1": { + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-2": { + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-3": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "il-central-1": { + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_artifact_compression_type": "None", + "dynamic_container_deployment_supported": false + }, + "variants": { + "public_model_no_additional_sources": {}, + "public_model_public_additional_sources": { + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/draft/model/" + }, + "provider": { + "name": "JumpStart", + "classification": "ungated" + } + } + ] + } + }, + "gated_model_gated_additional_sources": { + "hosting_eula_key": "fmhMetadata/eula/gatedModelEula.txt", + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "gated_draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/gated/draft/" + }, + "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", + "provider": { + "name": "JumpStart", + "classification": "gated" + } + } + ] + } + }, + "public_model_mixed_additional_sources": { + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/draft/model/" + }, + "provider": { + "name": "JumpStart", + "classification": "ungated" + } + }, + { + "channel_name": "gated_draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/gated/draft/" + }, + "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", + "provider": { + "name": "JumpStart", + "classification": "gated" + } + } + ] + } + } + } +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 03f8c908c4..8a6471133f 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1,5 +1,6 @@ """Unit tests for ModelBuilderServers class.""" +import json import os import sys import tempfile @@ -1004,460 +1005,204 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init mock_create.assert_called_once() - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_propagates_additional_model_data_sources( - self, mock_create, mock_prepare, mock_init - ): - """additional_model_data_sources from the JumpStart spec are propagated - to the builder with the JumpStart-internal HostingEulaKey metadata - stripped (mirroring _add_model_access_configs_to_model_data_sources).""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - # JumpStart spec includes an internal metadata key (HostingEulaKey) that - # the CreateModel API does not accept; it must be stripped. - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - - self.builder._build_for_jumpstart() - self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - } - ], +class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): + """User-flow tests for additional model data sources, driven by raw spec JSON. + + Unlike the Mock-based tests above, these tests exercise the code the way a + user calls it: + + builder = ModelBuilder(model="", ...) + builder.accept_eula = True # the documented EULA knob + builder.build() + + Only the JSON-fetch boundary is mocked (the spec and manifest accessors, + fed from tests/unit/servers/data/jumpstart_specs.json) plus the AWS + boundary (_prepare_for_mode / _create_model). Everything in between -- + JumpStartModelSpecs parsing, the factory shaping in get_init_kwargs + (bucket resolution, PascalCase conversion), JumpStart model-id detection, + and _build_for_jumpstart's EULA handling -- runs for real. + + The fixture file holds one raw snake_case spec per use case (the shape + published to the JumpStart content bucket): public model without + additional sources, public model with a public source, gated model with a + gated source, and a public model with mixed public+gated sources. + """ + + SPEC_FIXTURE_PATH = Path(__file__).parent / "data" / "jumpstart_specs.json" + MODEL_ID = "pytorch-ic-mobilenet-v2" + + EXPECTED_PUBLIC_SOURCE = { + "ChannelName": "draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", + }, + } + EXPECTED_GATED_SOURCE_ACCEPTED = { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/", + "ModelAccessConfig": {"AcceptEula": True}, + }, + } + + @classmethod + def setUpClass(cls): + with open(cls.SPEC_FIXTURE_PATH) as f: + cls.spec_fixture = json.load(f) + + def _spec_json(self, variant): + """Returns the raw spec JSON for a named fixture variant.""" + spec = json.loads(json.dumps(self.spec_fixture["base_spec"])) + spec.update(self.spec_fixture["variants"][variant]) + return spec + + def _make_builder(self, variant, accept_eula=None): + """Builds a real ModelBuilder the way a user would, with the JSON-fetch + boundary patched to serve the fixture variant.""" + from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor + from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs + from sagemaker.serve.model_builder import ModelBuilder + + spec_json = self._spec_json(variant) + + session = Mock() + session.boto_region_name = "us-west-2" + session.get_caller_identity_arn = Mock( + return_value="arn:aws:iam::123456789012:role/SageMakerRole" ) + session.sagemaker_config = {} + session.config = None - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_without_additional_model_data_sources( - self, mock_create, mock_prepare, mock_init - ): - """Models whose spec has no additional_model_data_sources are unaffected.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = None - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - - self.builder._build_for_jumpstart() - - self.assertIsNone( - getattr(self.builder, "additional_model_data_sources", None) - ) - - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_gated_additional_model_data_source_with_accept_eula( - self, mock_create, mock_prepare, mock_init - ): - """A gated additional source (truthy HostingEulaKey) has the acceptance - folded into its ModelAccessConfig when accept_eula is set, mirroring the - base model's EULA handling.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ + header = JumpStartModelHeader( { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + "model_id": self.MODEL_ID, + "version": "1.0.0", + "min_version": "2.49.0", + "spec_key": f"community_models_specs/{self.MODEL_ID}/specs_v1.0.0.json", } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True - - self.builder._build_for_jumpstart() - - self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - "ModelAccessConfig": {"AcceptEula": True}, - }, - } - ], ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_gated_additional_model_data_source_without_accept_eula_raises( - self, mock_create, mock_prepare, mock_init - ): - """A gated additional source without accept_eula raises, matching the - base model's EULA enforcement.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", - } + patchers = [ + patch.object( + JumpStartModelsAccessor, + "get_model_specs", + side_effect=lambda *a, **kw: JumpStartModelSpecs( + json.loads(json.dumps(spec_json)) + ), + ), + patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=[header]), + patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), + patch.object(ModelBuilder, "_create_model", return_value=Mock()), ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = None + for p in patchers: + p.start() + self.addCleanup(p.stop) + + builder = ModelBuilder( + model=self.MODEL_ID, + mode=Mode.SAGEMAKER_ENDPOINT, + role_arn="arn:aws:iam::123456789012:role/SageMakerRole", + sagemaker_session=session, + ) + if accept_eula is not None: + builder.accept_eula = accept_eula + return builder - with self.assertRaises(ValueError) as ctx: - self.builder._build_for_jumpstart() - self.assertIn("accept_eula must be set to True", str(ctx.exception)) + def test_public_model_without_additional_sources(self): + """A spec with no additional data sources builds cleanly and sets none.""" + builder = self._make_builder("public_model_no_additional_sources") - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_multiple_public_additional_model_data_sources_pass_through( - self, mock_create, mock_prepare, mock_init - ): - """Multiple ungated additional sources are all propagated with only the - HostingEulaKey metadata stripped.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - }, - { - "ChannelName": "scripts", - "S3DataSource": { - "S3Uri": "s3://bucket/scripts/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - }, - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None + builder.build() - self.builder._build_for_jumpstart() + self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) + + def test_public_additional_source_builds_without_accept_eula(self): + """A public (ungated) additional source deploys without the user ever + touching accept_eula; the internal HostingEulaKey is stripped.""" + builder = self._make_builder("public_model_public_additional_sources") + + builder.build() self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - }, - { - "ChannelName": "scripts", - "S3DataSource": { - "S3Uri": "s3://bucket/scripts/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - }, - ], + builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_public_additional_model_data_source_ignores_accept_eula( - self, mock_create, mock_prepare, mock_init - ): - """accept_eula on an ungated additional source is ignored: no - ModelAccessConfig is added (mirrors the v2 helper, which only applies - access configs to gated sources).""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True + def test_public_additional_source_unaffected_by_accept_eula(self): + """Passing accept_eula=True for a public source is harmless: no + ModelAccessConfig is added.""" + builder = self._make_builder( + "public_model_public_additional_sources", accept_eula=True + ) - self.builder._build_for_jumpstart() + builder.build() self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - } - ], + builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_mixed_additional_model_data_sources_with_accept_eula( - self, mock_create, mock_prepare, mock_init - ): - """With a mix of ungated and gated sources and accept_eula=True, the - ungated source passes through untouched while the gated source gets - ModelAccessConfig folded into its S3DataSource.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - }, - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", - }, - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True + def test_gated_additional_source_with_accept_eula_builds(self): + """User sets accept_eula=True on the builder: the gated source deploys + with the acceptance recorded in its ModelAccessConfig.""" + builder = self._make_builder( + "gated_model_gated_additional_sources", accept_eula=True + ) - self.builder._build_for_jumpstart() + builder.build() self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - }, - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - "ModelAccessConfig": {"AcceptEula": True}, - }, - }, - ], + builder.additional_model_data_sources, [self.EXPECTED_GATED_SOURCE_ACCEPTED] ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_gated_additional_model_data_source_accept_eula_false_raises( - self, mock_create, mock_prepare, mock_init - ): - """An explicit accept_eula=False on a gated additional source raises, - matching the v2 helper's rejection of a false acceptance.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", - } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = False + def test_gated_additional_source_without_accept_eula_raises(self): + """User never sets accept_eula: build() fails with the actionable + EULA error instead of silently deploying the gated source.""" + builder = self._make_builder("gated_model_gated_additional_sources") with self.assertRaises(ValueError) as ctx: - self.builder._build_for_jumpstart() + builder.build() + self.assertIn("accept_eula must be set to True", str(ctx.exception)) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_additional_model_data_sources_from_raw_spec_json( - self, mock_create, mock_prepare, mock_init - ): - """End-to-end from a raw JumpStart spec JSON: the real - JumpStartModelSpecs parsing and the real factory shaping helper - (_add_additional_model_data_sources_to_kwargs) produce the API-shape - init kwargs, and _build_for_jumpstart translates them into the final - CreateModel shape (HostingEulaKey stripped, EULA acceptance folded - into the gated source's ModelAccessConfig).""" - from sagemaker.core.jumpstart.factory.utils import ( - _add_additional_model_data_sources_to_kwargs, + def test_gated_additional_source_with_accept_eula_false_raises(self): + """User explicitly sets accept_eula=False: build() fails the same way.""" + builder = self._make_builder( + "gated_model_gated_additional_sources", accept_eula=False ) - from sagemaker.core.jumpstart.types import JumpStartModelSpecs - - # Raw spec JSON in the snake_case shape published to the JumpStart - # content bucket (hosting_additional_data_sources.speculative_decoding). - raw_spec_json = { - "model_id": "pytorch-ic-mobilenet-v2", - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/draft/model/", - }, - "provider": {"name": "JumpStart", "classification": "ungated"}, - }, - { - "channel_name": "gated_draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/gated/draft/", - }, - "hosting_eula_key": "fmhMetadata/eula/llama3Eula.txt", - "provider": {"name": "JumpStart", "classification": "gated"}, - }, - ] - }, - } - - # Run the real factory shaping over the real parsed spec, exactly as - # get_init_kwargs does, to produce the API-shape init kwargs. - mock_init_kwargs = Mock() - mock_init_kwargs.specs = JumpStartModelSpecs(raw_spec_json) - mock_init_kwargs.region = "us-west-2" - mock_init_kwargs.additional_model_data_sources = None - mock_init_kwargs = _add_additional_model_data_sources_to_kwargs( - kwargs=mock_init_kwargs + + with self.assertRaises(ValueError) as ctx: + builder.build() + + self.assertIn("accept_eula must be set to True", str(ctx.exception)) + + def test_mixed_additional_sources_with_accept_eula(self): + """Public + gated sources together with accept_eula=True: the public + source passes through untouched, the gated source gets the + ModelAccessConfig.""" + builder = self._make_builder( + "public_model_mixed_additional_sources", accept_eula=True ) - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init.return_value = mock_init_kwargs - # The factory shaping keeps HostingEulaKey (internal metadata) in the - # API-shape kwargs; that is what _build_for_jumpstart must translate. + builder.build() + self.assertEqual( - [s.get("HostingEulaKey") for s in mock_init_kwargs.additional_model_data_sources], - [None, "fmhMetadata/eula/llama3Eula.txt"], + builder.additional_model_data_sources, + [self.EXPECTED_PUBLIC_SOURCE, self.EXPECTED_GATED_SOURCE_ACCEPTED], ) - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True + def test_mixed_additional_sources_without_accept_eula_raises(self): + """Public + gated sources together without accept_eula: the gated + source blocks the build.""" + builder = self._make_builder("public_model_mixed_additional_sources") - self.builder._build_for_jumpstart() + with self.assertRaises(ValueError) as ctx: + builder.build() - self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "CompressionType": "None", - "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", - }, - }, - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "CompressionType": "None", - "S3DataType": "S3Prefix", - "S3Uri": ( - "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/" - ), - "ModelAccessConfig": {"AcceptEula": True}, - }, - }, - ], - ) + self.assertIn("accept_eula must be set to True", str(ctx.exception)) class TestDeployWrappers(unittest.TestCase): From bdb7de526b32fbcc679f956ba2174d15072b52fd Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 14:59:02 +0000 Subject: [PATCH 4/8] test(serve): use real captured JumpStart specs as test fixtures Replace the synthetic base-spec-plus-variants fixture with real, unmodified specs captured from the production JumpStart content bucket, mirroring master-v2's PROTOTYPICAL_MODEL_SPECS_DICT pattern: the spec accessor routes by model id into one captured spec file per model, and each test calls the SDK exactly like a customer does. Fixture models (tests/unit/servers/data/jumpstart_specs/): - pytorch-ic-mobilenet-v2 v3.0.25: public, no additional data sources - openai-reasoning-gpt-oss-20b v3.38.0: ungated EAGLE speculative decoding source in its default (lmi-optimized) config -- the model that surfaced the propagation bug - meta-textgeneration-llama-3-1-70b v2.16.2: gated draft_model source (hosting_eula_key) in its lmi-optimized config, selected in tests via the public set_deployment_config API All patching is centralized in setUp (spec fetch, manifest, and the AWS boundary), so test bodies contain only public API calls: ModelBuilder(model=, instance_type=...), accept_eula, and build(). Real inference-config resolution now runs in every test. --- .../unit/servers/data/jumpstart_specs.json | 898 ---- .../data/jumpstart_specs/manifest.json | 38 + .../meta-textgeneration-llama-3-1-70b.json | 1910 +++++++++ .../openai-reasoning-gpt-oss-20b.json | 3801 +++++++++++++++++ .../pytorch-ic-mobilenet-v2.json | 1093 +++++ .../servers/test_model_builder_servers.py | 264 +- 6 files changed, 6990 insertions(+), 1014 deletions(-) delete mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json deleted file mode 100644 index dfedcde98e..0000000000 --- a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json +++ /dev/null @@ -1,898 +0,0 @@ -{ - "_comment": "Raw JumpStart spec JSON fixtures for user-flow tests. base_spec is the prototypical pytorch-ic-mobilenet-v2 spec (same prototype the pre-v3 test suite used); each variant is an overlay merged over base_spec by the test loader. The shapes mirror what is published to the JumpStart content bucket (snake_case, hosting_additional_data_sources.speculative_decoding).", - "base_spec": { - "model_id": "pytorch-ic-mobilenet-v2", - "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", - "version": "3.0.6", - "min_sdk_version": "2.189.0", - "training_supported": true, - "incremental_training_supported": true, - "hosting_ecr_specs": { - "framework": "pytorch", - "framework_version": "1.10.0", - "py_version": "py38" - }, - "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", - "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", - "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", - "hosting_prepacked_artifact_version": "1.0.0", - "hosting_use_script_uri": false, - "inference_vulnerable": false, - "inference_dependencies": [], - "inference_vulnerabilities": [], - "training_vulnerable": false, - "training_dependencies": [ - "sagemaker_jumpstart_prepack_script_utilities==1.0.0" - ], - "training_vulnerabilities": [], - "deprecated": false, - "hyperparameters": [ - { - "name": "train_only_top_layer", - "type": "text", - "default": "True", - "options": [ - "True", - "False" - ], - "scope": "algorithm" - }, - { - "name": "epochs", - "type": "int", - "default": 5, - "min": 1, - "max": 1000, - "scope": "algorithm" - }, - { - "name": "learning_rate", - "type": "float", - "default": 0.001, - "min": 1e-08, - "max": 1, - "scope": "algorithm" - }, - { - "name": "batch_size", - "type": "int", - "default": 4, - "min": 1, - "max": 1024, - "scope": "algorithm" - }, - { - "name": "reinitialize_top_layer", - "type": "text", - "default": "Auto", - "options": [ - "Auto", - "True", - "False" - ], - "scope": "algorithm" - }, - { - "name": "sagemaker_submit_directory", - "type": "text", - "default": "/opt/ml/input/data/code/sourcedir.tar.gz", - "scope": "container" - }, - { - "name": "sagemaker_program", - "type": "text", - "default": "transfer_learning.py", - "scope": "container" - }, - { - "name": "sagemaker_container_log_level", - "type": "text", - "default": "20", - "scope": "container" - } - ], - "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", - "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", - "training_prepacked_script_version": "1.1.0", - "training_ecr_specs": { - "framework": "pytorch", - "framework_version": "1.10.0", - "py_version": "py38" - }, - "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", - "inference_environment_variables": [ - { - "name": "SAGEMAKER_PROGRAM", - "type": "text", - "default": "inference.py", - "scope": "container", - "required_for_model_class": true - }, - { - "name": "SAGEMAKER_SUBMIT_DIRECTORY", - "type": "text", - "default": "/opt/ml/model/code", - "scope": "container", - "required_for_model_class": false - }, - { - "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", - "type": "text", - "default": "20", - "scope": "container", - "required_for_model_class": false - }, - { - "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", - "type": "text", - "default": "3600", - "scope": "container", - "required_for_model_class": false - }, - { - "name": "ENDPOINT_SERVER_TIMEOUT", - "type": "int", - "default": 3600, - "scope": "container", - "required_for_model_class": true - }, - { - "name": "MODEL_CACHE_ROOT", - "type": "text", - "default": "/opt/ml/model", - "scope": "container", - "required_for_model_class": true - }, - { - "name": "SAGEMAKER_ENV", - "type": "text", - "default": "1", - "scope": "container", - "required_for_model_class": true - }, - { - "name": "SAGEMAKER_MODEL_SERVER_WORKERS", - "type": "int", - "default": 1, - "scope": "container", - "required_for_model_class": true - } - ], - "metrics": [ - { - "Name": "pytorch-ic:val-accuracy", - "Regex": "val_accuracy: ([0-9\\.]+)" - } - ], - "default_inference_instance_type": "ml.m5.large", - "supported_inference_instance_types": [ - "ml.m5.large", - "ml.m5.xlarge", - "ml.c5.xlarge", - "ml.c5.2xlarge", - "ml.m4.large", - "ml.m4.xlarge" - ], - "default_training_instance_type": "ml.m5.xlarge", - "supported_training_instance_types": [ - "ml.m5.xlarge", - "ml.c5.2xlarge", - "ml.m4.xlarge" - ], - "model_kwargs": {}, - "deploy_kwargs": {}, - "estimator_kwargs": { - "encrypt_inter_container_traffic": true, - "max_run": 360000 - }, - "fit_kwargs": {}, - "predictor_specs": { - "supported_content_types": [ - "application/x-image" - ], - "supported_accept_types": [ - "application/json;verbose", - "application/json" - ], - "default_content_type": "application/x-image", - "default_accept_type": "application/json" - }, - "inference_enable_network_isolation": true, - "training_enable_network_isolation": true, - "default_training_dataset_key": "training-datasets/tf_flowers/", - "validation_supported": false, - "fine_tuning_supported": true, - "resource_name_base": "pt-ic-mobilenet-v2", - "hosting_instance_type_variants": { - "regional_aliases": { - "af-south-1": { - "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-east-1": { - "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-northeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-northeast-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-northeast-3": { - "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-south-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-3": { - "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-5": { - "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ca-central-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "cn-north-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" - }, - "cn-northwest-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-central-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-central-2": { - "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-north-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-south-1": { - "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-west-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-west-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-west-3": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "il-central-1": { - "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "me-central-1": { - "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "me-south-1": { - "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "sa-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-east-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-gov-east-1": { - "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-gov-west-1": { - "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-west-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-west-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - } - }, - "variants": { - "c4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5n": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "g4dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6e": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "local": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "local_gpu": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "m4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "p2": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4d": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4de": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "r5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t2": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t3": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "trn1": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - }, - "trn1n": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - } - } - }, - "training_instance_type_variants": { - "regional_aliases": { - "af-south-1": { - "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-east-1": { - "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-northeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-northeast-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-northeast-3": { - "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-south-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-3": { - "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-5": { - "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ca-central-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "cn-north-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" - }, - "cn-northwest-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" - }, - "eu-central-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-central-2": { - "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-north-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-south-1": { - "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-west-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-west-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-west-3": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "il-central-1": { - "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "me-central-1": { - "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "me-south-1": { - "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "sa-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-east-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-gov-east-1": { - "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-gov-west-1": { - "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-west-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-west-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - } - }, - "variants": { - "c4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5n": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "g4dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6e": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "local": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "local_gpu": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "m4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "p2": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4d": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4de": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "r5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t2": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t3": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "trn1": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - }, - "trn1n": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - } - } - }, - "hosting_artifact_s3_data_type": "S3Prefix", - "hosting_artifact_compression_type": "None", - "dynamic_container_deployment_supported": false - }, - "variants": { - "public_model_no_additional_sources": {}, - "public_model_public_additional_sources": { - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/draft/model/" - }, - "provider": { - "name": "JumpStart", - "classification": "ungated" - } - } - ] - } - }, - "gated_model_gated_additional_sources": { - "hosting_eula_key": "fmhMetadata/eula/gatedModelEula.txt", - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "gated_draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/gated/draft/" - }, - "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", - "provider": { - "name": "JumpStart", - "classification": "gated" - } - } - ] - } - }, - "public_model_mixed_additional_sources": { - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/draft/model/" - }, - "provider": { - "name": "JumpStart", - "classification": "ungated" - } - }, - { - "channel_name": "gated_draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/gated/draft/" - }, - "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", - "provider": { - "name": "JumpStart", - "classification": "gated" - } - } - ] - } - } - } -} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json new file mode 100644 index 0000000000..486ca44a27 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json @@ -0,0 +1,38 @@ +[ + { + "model_id": "pytorch-ic-mobilenet-v2", + "version": "3.0.25", + "min_version": "2.189.0", + "spec_key": "community_models/pytorch-ic-mobilenet-v2/specs_v3.0.25.json", + "deprecated": false, + "provider": "pytorch", + "search_keywords": [ + "MobileNet V2", + "Vision", + "ResNet 18" + ] + }, + { + "model_id": "openai-reasoning-gpt-oss-20b", + "version": "3.38.0", + "min_version": "2.244.2", + "spec_key": "community_models/openai-reasoning-gpt-oss-20b/specs_v3.38.0.json", + "deprecated": false, + "provider": "openai", + "search_keywords": [ + "Text Generation", + "Reasoning" + ] + }, + { + "model_id": "meta-textgeneration-llama-3-1-70b", + "version": "2.16.2", + "min_version": "2.232.1", + "spec_key": "community_models/meta-textgeneration-llama-3-1-70b/specs_v2.16.2.json", + "deprecated": false, + "provider": "meta", + "search_keywords": [ + "Text Generation" + ] + } +] \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json new file mode 100644 index 0000000000..3b53a2e449 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json @@ -0,0 +1,1910 @@ +{ + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_training_dataset_key": "training-datasets/sec_amazon/", + "default_training_instance_type": "ml.g5.48xlarge", + "deprecated": false, + "dynamic_container_deployment_supported": true, + "estimator_kwargs": { + "disable_output_compression": true, + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fine_tuning_supported": true, + "fit_kwargs": {}, + "gated_bucket": true, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_eula_key": "fmhMetadata/eula/llama3_1Eula.txt", + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "huggingface_id": "meta-llama/Llama-3.1-70B", + "hyperparameters": [ + { + "default": 5, + "max": 1000, + "min": 1, + "name": "epoch", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.0001, + "max": 1, + "min": 1e-08, + "name": "learning_rate", + "scope": "algorithm", + "type": "float" + }, + { + "default": 8, + "min": 1, + "name": "lora_r", + "scope": "algorithm", + "type": "int" + }, + { + "default": 32, + "min": 1, + "name": "lora_alpha", + "scope": "algorithm", + "type": "int" + }, + { + "default": "q_proj,v_proj", + "name": "target_modules", + "scope": "algorithm", + "type": "text" + }, + { + "default": 0.05, + "max": 1, + "min": 0, + "name": "lora_dropout", + "scope": "algorithm", + "type": "float" + }, + { + "default": "False", + "name": "instruction_tuned", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "False", + "name": "chat_dataset", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "True", + "name": "add_input_output_demarcation_key", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": 1, + "max": 1000, + "min": 1, + "name": "per_device_train_batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": 1, + "max": 1000, + "min": 1, + "name": "per_device_eval_batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_train_samples", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_val_samples", + "scope": "algorithm", + "type": "int" + }, + { + "default": 10, + "max": 1000, + "min": 1, + "name": "seed", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_input_length", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.2, + "max": 1, + "min": 0, + "name": "validation_split_ratio", + "scope": "algorithm", + "type": "float" + }, + { + "default": 0, + "min": 0, + "name": "train_data_split_seed", + "scope": "algorithm", + "type": "int" + }, + { + "default": "None", + "name": "preprocessing_num_workers", + "scope": "algorithm", + "type": "text" + }, + { + "default": "True", + "name": "int8_quantization", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "False", + "name": "enable_fsdp", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "Llama3.1", + "name": "chat_template", + "scope": "algorithm", + "type": "text" + }, + { + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "name": "sagemaker_submit_directory", + "scope": "container", + "type": "text" + }, + { + "default": "transfer_learning.py", + "name": "sagemaker_program", + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "sagemaker_container_log_level", + "scope": "container", + "type": "text" + } + ], + "incremental_training_supported": false, + "inference_config_components": { + "lmi": { + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_payloads": { + "englishToFrench": { + "body": { + "inputs": "Translate English to French:\nsea otter => loutre de mer\npeppermint => menthe poivr\u00e9e\nplush girafe => girafe peluche\ncheese =>", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "meaningOfLife": { + "body": { + "inputs": "I believe the meaning of life is", + "parameters": { + "details": true, + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "teamMessage": { + "body": { + "inputs": "A brief message congratulating the team on the launch:\n\nHi everyone,\n\nI just ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "theoryOfRelativity": { + "body": { + "inputs": "Simply put, the theory of relativity states that ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.p4d.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + } + } + }, + "ml.p4de.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + }, + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "true", + "name": "OPTION_ENFORCE_EAGER", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge", + "ml.g6.48xlarge", + "ml.p5.48xlarge" + ] + }, + "lmi-optimized": { + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_payloads": { + "englishToFrench": { + "body": { + "inputs": "Translate English to French:\nsea otter => loutre de mer\npeppermint => menthe poivr\u00e9e\nplush girafe => girafe peluche\ncheese =>", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "meaningOfLife": { + "body": { + "inputs": "I believe the meaning of life is", + "parameters": { + "details": true, + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "teamMessage": { + "body": { + "inputs": "A brief message congratulating the team on the launch:\n\nHi everyone,\n\nI just ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "theoryOfRelativity": { + "body": { + "inputs": "Simply put, the theory of relativity states that ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "artifact_version": "v1.0.0", + "channel_name": "draft_model", + "hosting_eula_key": "fmhMetadata/eula/llama3_2Eula.txt", + "provider": { + "classification": "gated", + "name": "JumpStart" + }, + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "meta-textgeneration/meta-textgeneration-llama-3-2-1b/artifacts/inference-prepack/v1.0.0/" + } + } + ] + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.p4d.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + } + } + }, + "ml.p4de.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + }, + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/additional-model-data-sources/draft_model", + "name": "OPTION_SPECULATIVE_DRAFT_MODEL", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.g6.48xlarge", + "ml.p4d.24xlarge", + "ml.p5.48xlarge" + ] + } + }, + "inference_config_rankings": { + "overall": { + "description": "default", + "rankings": [ + "lmi", + "lmi-optimized" + ] + } + }, + "inference_configs": { + "lmi": { + "benchmark_metrics": { + "ml.g5.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "2.68" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "18.6" + } + ], + "ml.g6.48xlarge": [ + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "1.41" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "4.9" + } + ], + "ml.p4d.24xlarge": [ + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "0.11" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "37.4" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "0.13" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "21.6" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "0.16" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "13.9" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "0.20" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "8.4" + }, + { + "concurrency": "128", + "name": "latency", + "unit": "sec", + "value": "3.91" + }, + { + "concurrency": "128", + "name": "throughput", + "unit": "tokens/sec", + "value": "7.6" + } + ], + "ml.p5.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "48.3" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "48.3" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "46.3" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "42.1" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "0.09" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "35.8" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "0.09" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "26.6" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "0.10" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "17.3" + } + ] + }, + "component_names": [ + "lmi" + ] + }, + "lmi-optimized": { + "acceleration_configs": [ + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "TRT-LLM 0.11.0 in LMI v11 does not support llama 3.1" + } + }, + "enabled": false, + "type": "Compilation" + }, + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "LMI v11 does not support Speculative Decoding for TRT" + } + }, + "enabled": true, + "type": "Speculative-Decoding" + }, + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "TRT-LLM 0.11.0 in LMI v11 does not support llama 3.1" + } + }, + "enabled": false, + "type": "Quantization" + } + ], + "benchmark_metrics": { + "ml.g5.48xlarge": [ + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "2.94" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "16.1" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "4.04" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "6.1" + } + ], + "ml.g6.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.93" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "23.5" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "1.14" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "17.7" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "1.15" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "11.2" + } + ], + "ml.p4d.24xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.10" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "89.4" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "0.13" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "81.2" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "0.15" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "49.1" + } + ] + }, + "component_names": [ + "lmi-optimized" + ] + } + }, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "true", + "name": "OPTION_ENFORCE_EAGER", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "input_modalities": [ + "Text" + ], + "languages": [ + "English", + "German", + "French", + "Italian", + "Portuguese", + "Hindi", + "Spanish", + "Thai" + ], + "license": "Llama 3.1 Community License", + "metrics": [ + { + "Name": "huggingface-textgeneration:eval-loss", + "Regex": "eval_epoch_loss=tensor\\(([0-9\\.]+)" + }, + { + "Name": "huggingface-textgeneration:eval-ppl", + "Regex": "eval_ppl=tensor\\(([0-9\\.]+)" + }, + { + "Name": "huggingface-textgeneration:train-loss", + "Regex": "train_epoch_loss=([0-9\\.]+)" + } + ], + "min_sdk_version": "2.232.1", + "ml_framework": "Meta", + "model_access": "Gated", + "model_id": "meta-textgeneration-llama-3-1-70b", + "model_kwargs": {}, + "model_size": "70B-100B", + "output_modalities": [ + "Text" + ], + "provider": "meta", + "resource_name_base": "meta-textgeneration-llama-3-1-70b", + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge", + "ml.g6.48xlarge", + "ml.p5.48xlarge" + ], + "supported_training_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge" + ], + "training_artifact_key": "meta-training/train-meta-textgeneration-llama-3-1-70b.tar.gz", + "training_dependencies": [ + "accelerate==0.33.0", + "bitsandbytes==0.39.1", + "black==23.7.0", + "brotli==1.0.9", + "datasets==2.14.1", + "docstring-parser==0.16", + "fire==0.5.0", + "huggingface-hub==0.24.2", + "inflate64==0.3.1", + "loralib==0.1.1", + "multivolumefile==0.2.3", + "mypy-extensions==1.0.0", + "nvidia-cublas-cu12==12.1.3.1", + "nvidia-cuda-cupti-cu12==12.1.105", + "nvidia-cuda-nvrtc-cu12==12.1.105", + "nvidia-cuda-runtime-cu12==12.1.105", + "nvidia-cudnn-cu12==8.9.2.26", + "nvidia-cufft-cu12==11.0.2.54", + "nvidia-curand-cu12==10.3.2.106", + "nvidia-cusolver-cu12==11.4.5.107", + "nvidia-cusolver-cu12==11.4.5.107", + "nvidia-cusparse-cu12==12.1.0.106", + "nvidia-nccl-cu12==2.19.3", + "nvidia-nvjitlink-cu12==12.3.101", + "nvidia-nvtx-cu12==12.1.105", + "pathspec==0.11.1", + "peft==0.4.0", + "py7zr==0.20.5", + "pybcj==1.0.1", + "pycryptodomex==3.18.0", + "pyppmd==1.0.0", + "pyzstd==0.15.9", + "safetensors==0.4.2", + "sagemaker_jumpstart_huggingface_script_utilities==1.2.7", + "sagemaker_jumpstart_script_utilities==1.1.9", + "scipy==1.11.1", + "shtab==1.7.1", + "termcolor==2.3.0", + "texttable==1.6.7", + "tokenize-rt==5.1.0", + "tokenizers==0.19.1", + "torch==2.2.0", + "transformers==4.43.1", + "triton==2.2.0", + "trl==0.8.1", + "typing-extensions==4.8.0", + "tyro==0.7.3" + ], + "training_ecr_specs": { + "framework": "huggingface", + "framework_version": "2.0.0", + "huggingface_transformers_version": "4.28.1", + "py_version": "py310" + }, + "training_enable_network_isolation": true, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "gpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-east-1": { + "gpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-3": { + "gpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-south-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-3": { + "gpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ca-central-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ca-west-1": { + "gpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "cn-north-1": { + "gpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "cn-northwest-1": { + "gpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-central-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-north-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-south-1": { + "gpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-3": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "il-central-1": { + "gpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "me-central-1": { + "gpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "me-south-1": { + "gpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "sa-east-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-east-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-east-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-gov-east-1": { + "gpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-gov-west-1": { + "gpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-iso-east-1": { + "gpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-isob-east-1": { + "gpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-west-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-west-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + } + }, + "variants": { + "g4dn": { + "properties": { + "gated_model_key_env_var_value": "meta-training/g4dn/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g5": { + "properties": { + "gated_model_key_env_var_value": "meta-training/g5/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p3dn": { + "properties": { + "gated_model_key_env_var_value": "meta-training/p3dn/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p4d": { + "properties": { + "gated_model_key_env_var_value": "meta-training/p4d/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + } + } + }, + "training_prepacked_script_key": "source-directory-tarballs/training/meta-textgeneration/prepack/inference-meta-textgeneration/v1.2.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.2.0", + "training_script_key": "source-directory-tarballs/training/meta-textgeneration/v1.2.0/sourcedir.tar.gz", + "training_supported": true, + "training_volume_size": 256, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://ai.meta.com/resources/models-and-libraries/llama-downloads/", + "validation_supported": true, + "version": "2.16.2" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json new file mode 100644 index 0000000000..b113d8bdff --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json @@ -0,0 +1,3801 @@ +{ + "context_window": "32K-128K", + "default_inference_instance_type": "ml.g7e.2xlarge", + "deprecated": false, + "dynamic_container_deployment_supported": true, + "fine_tuning_supported": false, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "vllm", + "framework_version": "0.18.0", + "py_version": "py312" + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "huggingface_id": "openai/gpt-oss-20b", + "incremental_training_supported": false, + "inference_config_components": { + "lmi": { + "default_inference_instance_type": "ml.g7e.2xlarge", + "default_payloads": { + "marketing": { + "body": { + "max_tokens": 200, + "messages": [ + { + "content": "Olly has just promoted you to the marketing department.", + "role": "system" + }, + { + "content": "Create ad text promoting Olly supplements for woman over 40 years using the following keywords: mental, health, sleep, energy, mood, cognitive, adult, children, supplement, and support. Include a slogan.", + "role": "user" + } + ], + "temperature": 0.7, + "top_p": 1 + }, + "content_type": "application/json" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.34.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-east-2": { + "alias_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-6": { + "alias_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-isoe-west-1": { + "alias_ecr_uri_1": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eusc-de-east-1": { + "alias_ecr_uri_1": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g7e.2xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "1" + }, + "resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5e.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5en.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + }, + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ] + }, + "lmi-optimized": { + "default_inference_instance_type": "ml.g7e.2xlarge", + "default_payloads": { + "marketing": { + "body": { + "max_tokens": 200, + "messages": [ + { + "content": "Olly has just promoted you to the marketing department.", + "role": "system" + }, + { + "content": "Create ad text promoting Olly supplements for woman over 40 years using the following keywords: mental, health, sleep, energy, mood, cognitive, adult, children, supplement, and support. Include a slogan.", + "role": "user" + } + ], + "temperature": 0.7, + "top_p": 1 + }, + "content_type": "application/json" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 3600, + "model_data_download_timeout": 3600 + }, + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "artifact_version": "v1.0.0", + "channel_name": "eagle", + "provider": { + "classification": "non-gated", + "name": "JumpStart" + }, + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "lmi-eagle-heads/models/gpt-oss-20b-p-eagle/" + } + } + ] + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "vllm", + "framework_version": "0.18.0", + "py_version": "py312" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-east-2": { + "alias_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-6": { + "alias_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-isoe-west-1": { + "alias_ecr_uri_1": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eusc-de-east-1": { + "alias_ecr_uri_1": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g7e.2xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "1" + }, + "resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5e.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5en.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + } + } + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + }, + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "8192", + "name": "SM_VLLM_MAX_MODEL_LEN", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "{\"model\": \"/opt/ml/additional-model-data-sources/eagle\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3, \"parallel_drafting\": true}", + "name": "SM_VLLM_SPECULATIVE_CONFIG", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SM_VLLM_TENSOR_PARALLEL_SIZE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ] + } + }, + "inference_config_rankings": { + "overall": { + "description": "default", + "rankings": [ + "lmi-optimized", + "lmi" + ] + } + }, + "inference_configs": { + "lmi": { + "component_names": [ + "lmi" + ] + }, + "lmi-optimized": { + "acceleration_configs": [ + { + "enabled": true, + "type": "Speculative-Decoding" + } + ], + "benchmark_metrics": { + "ml.g7e.2xlarge": [ + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "40" + }, + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "50.0" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "80" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "25.0" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "160" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "12.5" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "6.25" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "3.12" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "1.56" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "1.0" + }, + { + "concurrency": "128", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "128", + "name": "latency", + "unit": "sec", + "value": "1.0" + } + ] + }, + "component_names": [ + "lmi-optimized" + ] + } + }, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "8192", + "name": "SM_VLLM_MAX_MODEL_LEN", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "{\"model\": \"/opt/ml/additional-model-data-sources/eagle\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3, \"parallel_drafting\": true}", + "name": "SM_VLLM_SPECULATIVE_CONFIG", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SM_VLLM_TENSOR_PARALLEL_SIZE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "input_modalities": [ + "Text" + ], + "languages": [ + "English" + ], + "license": "Apache 2.0", + "metrics": [], + "min_sdk_version": "2.244.2", + "ml_framework": "PyTorch", + "model_access": "Open Weight", + "model_id": "openai-reasoning-gpt-oss-20b", + "model_kwargs": {}, + "model_size": "10B-70B", + "output_modalities": [ + "Text" + ], + "provider": "openai", + "recipe_collection": [ + { + "customization_technique": "DPO", + "display_name": "GPT OSS 20B Direct Preference Optimization on GPU, 4K sequence length", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_override_params_k8s_v2.2.4.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_payload_template_k8s_v2.2.4.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_dpo", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_dpo.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_override_params_sm_jobs_v2.2.4.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_payload_template_sm_jobs_v2.2.4.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.4" + ] + }, + { + "customization_technique": "SFT", + "display_name": "GPT OSS 20B Supervised Fine-Tuning with LoRA on GPU, 4K sequence length", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_override_params_k8s_v2.3.1.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_payload_template_k8s_v2.3.1.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_sft_lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_sft_lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_override_params_sm_jobs_v2.3.1.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_payload_template_sm_jobs_v2.3.1.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.3.1" + ] + }, + { + "customization_technique": "RLAIF", + "display_name": "GPT OSS 20B GRPO RLAIF Fine-Tuning with LoRA", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_override_params_k8s_v2.2.3.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_payload_template_k8s_v2.2.3.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "verl-grpo-rlaif-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlaif-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_override_params_sm_jobs_v2.2.3.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_payload_template_sm_jobs_v2.2.3.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.3" + ] + }, + { + "customization_technique": "RLVR", + "display_name": "GPT OSS 20B GRPO RLVR Fine-Tuning with LoRA", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_override_params_k8s_v2.2.2.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_payload_template_k8s_v2.2.2.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "verl-grpo-rlvr-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlvr-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "1K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_override_params_sm_jobs_v2.2.2.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_payload_template_sm_jobs_v2.2.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.2" + ] + }, + { + "display_name": "Open Source Evaluation Evaluation on GPU - Openai Reasoning Gpt Oss 20B", + "evaluation_type": "DeterministicEvaluation", + "hardware": "GPU", + "hp_eks_override_params_s3_key": "recipes/openai_reasoning_gpt_oss_20b_override_params_k8s_v2.0.2.json", + "hp_eks_payload_template_s3_key": "recipes/openai_reasoning_gpt_oss_20b_payload_template_k8s_v2.0.2.yaml", + "name": "open-source-eval-openai-reasoning-gpt-oss-20b-deterministic", + "regional_parameters": { + "gamma": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + }, + "prod": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + } + }, + "smtj_override_params_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-deterministic_override_params_sm_jobs_v2.0.2.json", + "smtj_recipe_template_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-deterministic_payload_template_sm_jobs_v2.0.2.yaml", + "supported_instance_types": [ + "ml.g5.12xlarge", + "ml.g5.24xlarge", + "ml.g5.48xlarge", + "ml.g6.24xlarge", + "ml.g6.48xlarge", + "ml.p4d.24xlarge", + "ml.p4de.24xlarge" + ], + "type": "Evaluation", + "versions": [ + "2.0.1" + ] + }, + { + "display_name": "Open Source Evaluation Evaluation on GPU - Openai Reasoning Gpt Oss 20B", + "evaluation_type": "LLMAJEvaluation", + "hardware": "CPU", + "hp_eks_override_params_s3_key": "recipes/openai_reasoning_gpt_oss_20b_override_params_k8s_v2.0.2.json", + "hp_eks_payload_template_s3_key": "recipes/openai_reasoning_gpt_oss_20b_payload_template_k8s_v2.0.2.yaml", + "name": "open-source-eval-openai-reasoning-gpt-oss-20b-llmaj", + "regional_parameters": { + "gamma": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + }, + "prod": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + } + }, + "smtj_override_params_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-llmaj_override_params_sm_jobs_v2.0.2.json", + "smtj_recipe_template_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-llmaj_payload_template_sm_jobs_v2.0.2.yaml", + "supported_instance_types": [ + "ml.t3.large" + ], + "type": "Evaluation", + "versions": [ + "2.0.1" + ] + }, + { + "customization_technique": "MTRL", + "display_name": "Agentic RFT for GPT OSS 20B Lora", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "name": "mtrl-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/mtrl-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-1": { + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-south-1": { + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-1": { + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-2": { + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-central-1": { + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-north-1": { + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-south-2": { + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-1": { + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-2": { + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "sa-east-1": { + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-east-1": { + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-east-2": { + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-west-1": { + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + } + }, + "sequence_length": "64K", + "smtj_override_params_s3_key": "recipes/mtrl-gpt-oss-20b-lora_override_params_sm_jobs_v1.0.22.json", + "smtj_recipe_template_s3_key": "recipes/mtrl-gpt-oss-20b-lora_payload_template_sm_jobs_v1.0.22.yaml", + "type": "FineTuning", + "versions": [ + "1.0.22" + ] + }, + { + "display_name": "MTRL Evaluation", + "evaluation_type": "MTRLEvaluation", + "name": "mtrl-eval-openai-reasoning-gpt-oss-20b", + "recipe_file_path": "recipes/evaluation/mtrl/mtrl_eval.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "prod": { + "us-east-1": { + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + } + }, + "smtj_override_params_s3_key": "recipes/mtrl-eval-openai-reasoning-gpt-oss-20b_override_params_sm_jobs_v1.0.12.json", + "smtj_recipe_template_s3_key": "recipes/mtrl-eval-openai-reasoning-gpt-oss-20b_payload_template_sm_jobs_v1.0.12.yaml", + "type": "Evaluation", + "versions": [ + "1.0.12" + ] + }, + { + "customization_technique": "SFT", + "display_name": "GPT-OSS 20B Supervised Fine-Tuning Full Fine-Tuning on GPU", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_override_params_k8s_v1.2.2.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_payload_template_k8s_v1.2.2.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_sft_fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_sft_fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_override_params_sm_jobs_v1.2.2.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_payload_template_sm_jobs_v1.2.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.2.2" + ] + }, + { + "customization_technique": "DPO", + "display_name": "GPT-OSS 20B Direct Preference Optimization Full Fine-Tuning on GPU", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_override_params_k8s_v1.3.2.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_payload_template_k8s_v1.3.2.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_dpo_fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_override_params_sm_jobs_v1.3.2.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_payload_template_sm_jobs_v1.3.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.3.2" + ] + }, + { + "customization_technique": "RLAIF", + "display_name": "GPT OSS 20B GRPO RLAIF Fine-Tuning", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_override_params_k8s_v1.2.3.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_payload_template_k8s_v1.2.3.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "verl-grpo-rlaif-gpt-oss-20b-fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlaif-gpt-oss-20b-fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_override_params_sm_jobs_v1.2.3.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_payload_template_sm_jobs_v1.2.3.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.2.3" + ] + }, + { + "customization_technique": "RLVR", + "display_name": "GPT OSS 20B GRPO RLVR Fine-Tuning", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_override_params_k8s_v1.3.2.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_payload_template_k8s_v1.3.2.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "verl-grpo-rlvr-gpt-oss-20b-fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlvr-gpt-oss-20b-fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "1K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_override_params_sm_jobs_v1.3.2.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_payload_template_sm_jobs_v1.3.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.3.2" + ] + } + ], + "resource_name_base": "openai-reasoning-gpt-oss-20b", + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ], + "training_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/training/v2.0.0/", + "training_dependencies": [], + "training_supported": false, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://huggingface.co/openai/gpt-oss-20b", + "validation_supported": true, + "version": "3.38.0" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json new file mode 100644 index 0000000000..16036c5025 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json @@ -0,0 +1,1093 @@ +{ + "default_inference_instance_type": "ml.m5.large", + "default_training_dataset_key": "training-datasets/tf_flowers/", + "default_training_instance_type": "ml.m5.xlarge", + "deploy_kwargs": {}, + "deprecated": false, + "dynamic_container_deployment_supported": false, + "estimator_kwargs": { + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fine_tuning_supported": true, + "fit_kwargs": {}, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-2": { + "cpu_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-2": { + "alias_ecr_uri_3": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-4": { + "alias_ecr_uri_3": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-6": { + "cpu_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-7": { + "cpu_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-west-1": { + "alias_ecr_uri_3": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-2": { + "alias_ecr_uri_3": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-2": { + "alias_ecr_uri_3": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-3": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "il-central-1": { + "alias_ecr_uri_3": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "mx-central-1": { + "cpu_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-iso-east-1": { + "cpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-inference:1.10.0-gpu-py38" + }, + "us-isob-east-1": { + "cpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "dl1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "i3en": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn2": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "hyperparameters": [ + { + "default": "True", + "name": "train_only_top_layer", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": 5, + "max": 1000, + "min": 1, + "name": "epochs", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.001, + "max": 1, + "min": 1e-08, + "name": "learning_rate", + "scope": "algorithm", + "type": "float" + }, + { + "default": 4, + "max": 1024, + "min": 1, + "name": "batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": "Auto", + "name": "reinitialize_top_layer", + "options": [ + "Auto", + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "name": "sagemaker_submit_directory", + "scope": "container", + "type": "text" + }, + { + "default": "transfer_learning.py", + "name": "sagemaker_program", + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "sagemaker_container_log_level", + "scope": "container", + "type": "text" + } + ], + "incremental_training_supported": true, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": false, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "metrics": [ + { + "Name": "pytorch-ic:val-accuracy", + "Regex": "val_accuracy: ([0-9\\.]+)" + } + ], + "min_sdk_version": "2.189.0", + "model_id": "pytorch-ic-mobilenet-v2", + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/x-image", + "supported_accept_types": [ + "application/json;verbose", + "application/json" + ], + "supported_content_types": [ + "application/x-image" + ] + }, + "provider": "pytorch", + "resource_name_base": "pt-ic-mobilenet-v2", + "supported_inference_instance_types": [ + "ml.m5.large", + "ml.m5.xlarge", + "ml.c5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "supported_training_instance_types": [ + "ml.m5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", + "training_dependencies": [ + "sagemaker_jumpstart_prepack_script_utilities==1.0.0" + ], + "training_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "training_enable_network_isolation": true, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-2": { + "cpu_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-2": { + "alias_ecr_uri_3": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-4": { + "alias_ecr_uri_3": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-6": { + "cpu_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-7": { + "cpu_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-west-1": { + "alias_ecr_uri_3": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-2": { + "alias_ecr_uri_3": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-2": { + "alias_ecr_uri_3": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-3": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "il-central-1": { + "alias_ecr_uri_3": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "mx-central-1": { + "cpu_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-iso-east-1": { + "cpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-training:1.10.0-gpu-py38" + }, + "us-isob-east-1": { + "cpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "dl1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "i3en": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn2": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.1.0", + "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", + "training_supported": true, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", + "validation_supported": false, + "version": "3.0.25" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 8a6471133f..f191335ee3 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1007,202 +1007,234 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): - """User-flow tests for additional model data sources, driven by raw spec JSON. + """User-flow tests for additional model data sources, driven by REAL specs. - Unlike the Mock-based tests above, these tests exercise the code the way a - user calls it: + Each test calls the SDK exactly the way a customer does: - builder = ModelBuilder(model="", ...) + builder = ModelBuilder(model="openai-reasoning-gpt-oss-20b", + instance_type="ml.g7e.2xlarge", ...) builder.accept_eula = True # the documented EULA knob builder.build() - Only the JSON-fetch boundary is mocked (the spec and manifest accessors, - fed from tests/unit/servers/data/jumpstart_specs.json) plus the AWS - boundary (_prepare_for_mode / _create_model). Everything in between -- - JumpStartModelSpecs parsing, the factory shaping in get_init_kwargs - (bucket resolution, PascalCase conversion), JumpStart model-id detection, - and _build_for_jumpstart's EULA handling -- runs for real. - - The fixture file holds one raw snake_case spec per use case (the shape - published to the JumpStart content bucket): public model without - additional sources, public model with a public source, gated model with a - gated source, and a public model with mixed public+gated sources. + The spec fixtures under tests/unit/servers/data/jumpstart_specs/ are real, + unmodified specs captured from the production JumpStart content bucket + (jumpstart-cache-prod-us-west-2): + + - pytorch-ic-mobilenet-v2: public model, no additional data sources. + - openai-reasoning-gpt-oss-20b: public model whose default (lmi-optimized) + config carries an ungated EAGLE speculative-decoding source -- the model + that surfaced the propagation bug this suite guards. + - meta-textgeneration-llama-3-1-70b: gated model whose lmi-optimized + config carries a GATED draft_model source + (hosting_eula_key=fmhMetadata/eula/llama3_2Eula.txt). + + Only two seams are patched, both once in setUp (the same seams the legacy + master-v2 JumpStartModel tests patched): + + 1. The spec-fetch boundary: JumpStartModelsAccessor.get_model_specs routes + by model id into the captured spec files (master-v2's + PROTOTYPICAL_MODEL_SPECS_DICT pattern), and _get_manifest serves the + captured manifest headers. + 2. The AWS boundary: _prepare_for_mode / _create_model and a mock session. + + Everything in between runs for real: JumpStart model-id detection, + JumpStartModelSpecs parsing, inference-config resolution, the + get_init_kwargs factory (image URI resolution, content-bucket injection, + PascalCase shaping), and _build_for_jumpstart's EULA translation. """ - SPEC_FIXTURE_PATH = Path(__file__).parent / "data" / "jumpstart_specs.json" - MODEL_ID = "pytorch-ic-mobilenet-v2" + SPEC_DIR = Path(__file__).parent / "data" / "jumpstart_specs" - EXPECTED_PUBLIC_SOURCE = { - "ChannelName": "draft_model", + ROLE_ARN = "arn:aws:iam::123456789012:role/SageMakerRole" + + # The real EAGLE source published in gpt-oss's default (lmi-optimized) + # config, after factory shaping: content bucket injected, PascalCase, and + # the null HostingEulaKey stripped by _build_for_jumpstart. + GPT_OSS_EAGLE_SOURCE = { + "ChannelName": "eagle", "S3DataSource": { "CompressionType": "None", "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", + "S3Uri": "s3://jumpstart-cache-prod-us-west-2/lmi-eagle-heads/models/gpt-oss-20b-p-eagle/", }, } - EXPECTED_GATED_SOURCE_ACCEPTED = { - "ChannelName": "gated_draft_model", + + # The real gated llama-3.2-1b draft model published in llama-3-1-70b's + # lmi-optimized config: private content bucket, EULA acceptance folded + # into ModelAccessConfig, HostingEulaKey stripped. + LLAMA_GATED_DRAFT_SOURCE_ACCEPTED = { + "ChannelName": "draft_model", "S3DataSource": { "CompressionType": "None", "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/", + "S3Uri": ( + "s3://jumpstart-private-cache-prod-us-west-2/meta-textgeneration/" + "meta-textgeneration-llama-3-2-1b/artifacts/inference-prepack/v1.0.0/" + ), "ModelAccessConfig": {"AcceptEula": True}, }, } - @classmethod - def setUpClass(cls): - with open(cls.SPEC_FIXTURE_PATH) as f: - cls.spec_fixture = json.load(f) - - def _spec_json(self, variant): - """Returns the raw spec JSON for a named fixture variant.""" - spec = json.loads(json.dumps(self.spec_fixture["base_spec"])) - spec.update(self.spec_fixture["variants"][variant]) - return spec - - def _make_builder(self, variant, accept_eula=None): - """Builds a real ModelBuilder the way a user would, with the JSON-fetch - boundary patched to serve the fixture variant.""" + def setUp(self): from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs from sagemaker.serve.model_builder import ModelBuilder - spec_json = self._spec_json(variant) + with open(self.SPEC_DIR / "manifest.json") as f: + manifest = [JumpStartModelHeader(header) for header in json.load(f)] - session = Mock() - session.boto_region_name = "us-west-2" - session.get_caller_identity_arn = Mock( - return_value="arn:aws:iam::123456789012:role/SageMakerRole" - ) - session.sagemaker_config = {} - session.config = None + spec_dir = self.SPEC_DIR - header = JumpStartModelHeader( - { - "model_id": self.MODEL_ID, - "version": "1.0.0", - "min_version": "2.49.0", - "spec_key": f"community_models_specs/{self.MODEL_ID}/specs_v1.0.0.json", - } - ) + def get_captured_model_specs(*args, **kwargs): + """Routes spec lookups by model id into the captured real spec + files, like master-v2's PROTOTYPICAL_MODEL_SPECS_DICT loaders.""" + model_id = kwargs.get("model_id") or args[1] + with open(spec_dir / f"{model_id}.json") as f: + return JumpStartModelSpecs(json.load(f)) patchers = [ patch.object( - JumpStartModelsAccessor, - "get_model_specs", - side_effect=lambda *a, **kw: JumpStartModelSpecs( - json.loads(json.dumps(spec_json)) - ), + JumpStartModelsAccessor, "get_model_specs", side_effect=get_captured_model_specs ), - patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=[header]), + patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=manifest), patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), patch.object(ModelBuilder, "_create_model", return_value=Mock()), ] - for p in patchers: - p.start() - self.addCleanup(p.stop) + for patcher in patchers: + patcher.start() + self.addCleanup(patcher.stop) + + session = Mock() + session.boto_region_name = "us-west-2" + session.get_caller_identity_arn = Mock(return_value=self.ROLE_ARN) + session.sagemaker_config = {} + session.config = None + self.session = session + + def _model_builder(self, model_id, instance_type=None): + """Constructs a ModelBuilder the way a customer does.""" + from sagemaker.serve.model_builder import ModelBuilder - builder = ModelBuilder( - model=self.MODEL_ID, + return ModelBuilder( + model=model_id, mode=Mode.SAGEMAKER_ENDPOINT, - role_arn="arn:aws:iam::123456789012:role/SageMakerRole", - sagemaker_session=session, + role_arn=self.ROLE_ARN, + sagemaker_session=self.session, + instance_type=instance_type, ) - if accept_eula is not None: - builder.accept_eula = accept_eula - return builder + + # ------------------------------------------------------------------ + # pytorch-ic-mobilenet-v2: public model, no additional data sources + # ------------------------------------------------------------------ def test_public_model_without_additional_sources(self): - """A spec with no additional data sources builds cleanly and sets none.""" - builder = self._make_builder("public_model_no_additional_sources") + """The base case: a model with no additional data sources builds + cleanly and never sets the attribute.""" + builder = self._model_builder("pytorch-ic-mobilenet-v2") builder.build() self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) - def test_public_additional_source_builds_without_accept_eula(self): - """A public (ungated) additional source deploys without the user ever - touching accept_eula; the internal HostingEulaKey is stripped.""" - builder = self._make_builder("public_model_public_additional_sources") + # ------------------------------------------------------------------ + # openai-reasoning-gpt-oss-20b: ungated EAGLE source in default config + # ------------------------------------------------------------------ + + def test_gpt_oss_eagle_source_propagates_without_accept_eula(self): + """The bug this suite guards: gpt-oss's default config ships an + ungated EAGLE draft model. A customer deploys it without ever touching + accept_eula, and the source must reach CreateModel.""" + builder = self._model_builder( + "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" + ) builder.build() self.assertEqual( - builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] + builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] ) - def test_public_additional_source_unaffected_by_accept_eula(self): - """Passing accept_eula=True for a public source is harmless: no - ModelAccessConfig is added.""" - builder = self._make_builder( - "public_model_public_additional_sources", accept_eula=True + def test_gpt_oss_eagle_source_unaffected_by_accept_eula(self): + """Setting accept_eula=True for an ungated source is harmless: no + ModelAccessConfig is added to it.""" + builder = self._model_builder( + "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" ) + builder.accept_eula = True builder.build() self.assertEqual( - builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] + builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] ) - def test_gated_additional_source_with_accept_eula_builds(self): - """User sets accept_eula=True on the builder: the gated source deploys - with the acceptance recorded in its ModelAccessConfig.""" - builder = self._make_builder( - "gated_model_gated_additional_sources", accept_eula=True + # ------------------------------------------------------------------ + # meta-textgeneration-llama-3-1-70b: gated draft_model source in the + # lmi-optimized config (selected via the public set_deployment_config API) + # ------------------------------------------------------------------ + + def test_llama_gated_draft_source_with_accept_eula_builds(self): + """A customer selects the lmi-optimized config and accepts the EULA: + the gated draft model deploys with acceptance recorded in its + ModelAccessConfig.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + ) + builder.set_deployment_config( + config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" ) + builder.accept_eula = True builder.build() self.assertEqual( - builder.additional_model_data_sources, [self.EXPECTED_GATED_SOURCE_ACCEPTED] + builder.additional_model_data_sources, + [self.LLAMA_GATED_DRAFT_SOURCE_ACCEPTED], ) - def test_gated_additional_source_without_accept_eula_raises(self): - """User never sets accept_eula: build() fails with the actionable - EULA error instead of silently deploying the gated source.""" - builder = self._make_builder("gated_model_gated_additional_sources") + def test_llama_gated_draft_source_without_accept_eula_raises(self): + """A customer selects the gated draft config but never accepts the + EULA: build() fails with the actionable error instead of silently + deploying the gated channel.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + ) + builder.set_deployment_config( + config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + ) with self.assertRaises(ValueError) as ctx: builder.build() self.assertIn("accept_eula must be set to True", str(ctx.exception)) - def test_gated_additional_source_with_accept_eula_false_raises(self): - """User explicitly sets accept_eula=False: build() fails the same way.""" - builder = self._make_builder( - "gated_model_gated_additional_sources", accept_eula=False + def test_llama_gated_draft_source_with_accept_eula_false_raises(self): + """A customer explicitly sets accept_eula=False: build() fails the + same way.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" ) + builder.set_deployment_config( + config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + ) + builder.accept_eula = False with self.assertRaises(ValueError) as ctx: builder.build() self.assertIn("accept_eula must be set to True", str(ctx.exception)) - def test_mixed_additional_sources_with_accept_eula(self): - """Public + gated sources together with accept_eula=True: the public - source passes through untouched, the gated source gets the - ModelAccessConfig.""" - builder = self._make_builder( - "public_model_mixed_additional_sources", accept_eula=True + def test_llama_default_config_has_no_additional_sources(self): + """The same gated model on its default (lmi) config carries no + additional sources: builds without accept_eula, nothing propagated.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" ) + builder.accept_eula = True builder.build() - self.assertEqual( - builder.additional_model_data_sources, - [self.EXPECTED_PUBLIC_SOURCE, self.EXPECTED_GATED_SOURCE_ACCEPTED], - ) - - def test_mixed_additional_sources_without_accept_eula_raises(self): - """Public + gated sources together without accept_eula: the gated - source blocks the build.""" - builder = self._make_builder("public_model_mixed_additional_sources") - - with self.assertRaises(ValueError) as ctx: - builder.build() - - self.assertIn("accept_eula must be set to True", str(ctx.exception)) + self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) class TestDeployWrappers(unittest.TestCase): From 09ee3e765795718c0d21fcc239eae358447d64d7 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 15:31:59 +0000 Subject: [PATCH 5/8] test(serve): assert the CreateModel request shape via the single-call API Restructure the additional-model-data-source tests on three axes: 1. Test names describe the case under test (gated source with accepted EULA sends ModelAccessConfig, unselected config sources do not leak, etc.), not the fixture model. 2. Each test drives the documented single-call customer API: ModelBuilder.from_jumpstart_config(JumpStartConfig(model_id, accept_eula, inference_config_name), compute=Compute(...)) followed by build(). No attribute pokes and no separate config-selection call. 3. Assertions target the CreateModel request itself: _create_model is no longer patched, so the real _create_sagemaker_model and _prepare_container_def_base run, and every test inspects the container_defs captured from sagemaker_session.create_model. This covers the full downstream impact: AdditionalModelDataSources presence/absence/shape, ModelAccessConfig on both the gated additional source and the primary ModelDataSource, and create_model never being called when the EULA gate raises. --- .../servers/test_model_builder_servers.py | 227 +++++++++--------- 1 file changed, 120 insertions(+), 107 deletions(-) diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index f191335ee3..cb49abaaf8 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1007,15 +1007,23 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): - """User-flow tests for additional model data sources, driven by REAL specs. + """End-to-end tests for additional model data sources, asserting the + CreateModel request shape. - Each test calls the SDK exactly the way a customer does: + Each test uses the documented single-call customer API: - builder = ModelBuilder(model="openai-reasoning-gpt-oss-20b", - instance_type="ml.g7e.2xlarge", ...) - builder.accept_eula = True # the documented EULA knob + builder = ModelBuilder.from_jumpstart_config( + jumpstart_config=JumpStartConfig( + model_id="...", accept_eula=True, inference_config_name="..."), + compute=Compute(instance_type="..."), ...) builder.build() + and asserts on what the SDK sends to the CreateModel API -- the + ``container_defs`` captured from ``sagemaker_session.create_model`` -- not + on builder internals. This is the request boto receives, so the assertions + cover the full downstream impact: primary container image, model data + source, and AdditionalModelDataSources. + The spec fixtures under tests/unit/servers/data/jumpstart_specs/ are real, unmodified specs captured from the production JumpStart content bucket (jumpstart-cache-prod-us-west-2): @@ -1028,19 +1036,22 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): config carries a GATED draft_model source (hosting_eula_key=fmhMetadata/eula/llama3_2Eula.txt). - Only two seams are patched, both once in setUp (the same seams the legacy - master-v2 JumpStartModel tests patched): + Patching is centralized in setUp on the same seams the legacy master-v2 + JumpStartModel tests patched: 1. The spec-fetch boundary: JumpStartModelsAccessor.get_model_specs routes by model id into the captured spec files (master-v2's - PROTOTYPICAL_MODEL_SPECS_DICT pattern), and _get_manifest serves the + PROTOTYPICAL_MODEL_SPECS_DICT pattern); _get_manifest serves the captured manifest headers. - 2. The AWS boundary: _prepare_for_mode / _create_model and a mock session. + 2. The AWS boundary: a mock session (whose create_model call is the + assertion target), IAM role validation, artifact staging + (_prepare_for_mode), and the post-create Model.get describe call. Everything in between runs for real: JumpStart model-id detection, JumpStartModelSpecs parsing, inference-config resolution, the get_init_kwargs factory (image URI resolution, content-bucket injection, - PascalCase shaping), and _build_for_jumpstart's EULA translation. + PascalCase shaping), _build_for_jumpstart's EULA translation, and + _prepare_container_def_base assembling the CreateModel request. """ SPEC_DIR = Path(__file__).parent / "data" / "jumpstart_specs" @@ -1048,9 +1059,9 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): ROLE_ARN = "arn:aws:iam::123456789012:role/SageMakerRole" # The real EAGLE source published in gpt-oss's default (lmi-optimized) - # config, after factory shaping: content bucket injected, PascalCase, and - # the null HostingEulaKey stripped by _build_for_jumpstart. - GPT_OSS_EAGLE_SOURCE = { + # config, as it must appear in the CreateModel request: content bucket + # injected, PascalCase, no HostingEulaKey (the API rejects it). + UNGATED_EAGLE_SOURCE = { "ChannelName": "eagle", "S3DataSource": { "CompressionType": "None", @@ -1060,9 +1071,10 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): } # The real gated llama-3.2-1b draft model published in llama-3-1-70b's - # lmi-optimized config: private content bucket, EULA acceptance folded - # into ModelAccessConfig, HostingEulaKey stripped. - LLAMA_GATED_DRAFT_SOURCE_ACCEPTED = { + # lmi-optimized config, as it must appear in the CreateModel request: + # private content bucket, EULA acceptance folded into ModelAccessConfig, + # no HostingEulaKey. + GATED_DRAFT_SOURCE_ACCEPTED = { "ChannelName": "draft_model", "S3DataSource": { "CompressionType": "None", @@ -1076,6 +1088,7 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): } def setUp(self): + import sagemaker.serve.model_builder as model_builder_module from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs from sagemaker.serve.model_builder import ModelBuilder @@ -1098,7 +1111,10 @@ def get_captured_model_specs(*args, **kwargs): ), patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=manifest), patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), - patch.object(ModelBuilder, "_create_model", return_value=Mock()), + patch.object( + model_builder_module, "resolve_and_validate_role", return_value=self.ROLE_ARN + ), + patch.object(model_builder_module.Model, "get", return_value=Mock()), ] for patcher in patchers: patcher.start() @@ -1109,132 +1125,129 @@ def get_captured_model_specs(*args, **kwargs): session.get_caller_identity_arn = Mock(return_value=self.ROLE_ARN) session.sagemaker_config = {} session.config = None + session.settings.include_jumpstart_tags = False + session.default_bucket = Mock(return_value="sagemaker-us-west-2-123456789012") + session.default_bucket_prefix = None + session.local_mode = False self.session = session - def _model_builder(self, model_id, instance_type=None): - """Constructs a ModelBuilder the way a customer does.""" + def _build(self, model_id, accept_eula=None, inference_config_name=None, instance_type=None): + """The customer flow, verbatim: one from_jumpstart_config call carrying + the model id, EULA decision, and config selection, then build().""" + from sagemaker.core.jumpstart.configs import JumpStartConfig + from sagemaker.core.training.configs import Compute from sagemaker.serve.model_builder import ModelBuilder - return ModelBuilder( - model=model_id, - mode=Mode.SAGEMAKER_ENDPOINT, + builder = ModelBuilder.from_jumpstart_config( + jumpstart_config=JumpStartConfig( + model_id=model_id, + accept_eula=accept_eula, + inference_config_name=inference_config_name, + ), role_arn=self.ROLE_ARN, + compute=Compute(instance_type=instance_type) if instance_type else None, sagemaker_session=self.session, - instance_type=instance_type, ) - - # ------------------------------------------------------------------ - # pytorch-ic-mobilenet-v2: public model, no additional data sources - # ------------------------------------------------------------------ - - def test_public_model_without_additional_sources(self): - """The base case: a model with no additional data sources builds - cleanly and never sets the attribute.""" - builder = self._model_builder("pytorch-ic-mobilenet-v2") - builder.build() - self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) + def _create_model_container_def(self): + """The container definition sent to the CreateModel API.""" + self.session.create_model.assert_called_once() + return self.session.create_model.call_args.kwargs["container_defs"] - # ------------------------------------------------------------------ - # openai-reasoning-gpt-oss-20b: ungated EAGLE source in default config - # ------------------------------------------------------------------ + def test_model_without_additional_sources_sends_none_to_create_model(self): + """Base case: the CreateModel request carries no + AdditionalModelDataSources field at all.""" + self._build("pytorch-ic-mobilenet-v2") - def test_gpt_oss_eagle_source_propagates_without_accept_eula(self): - """The bug this suite guards: gpt-oss's default config ships an - ungated EAGLE draft model. A customer deploys it without ever touching - accept_eula, and the source must reach CreateModel.""" - builder = self._model_builder( - "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" - ) + container_def = self._create_model_container_def() + self.assertNotIn("AdditionalModelDataSources", container_def) - builder.build() + def test_ungated_additional_source_reaches_create_model_without_eula(self): + """The bug this suite guards: an ungated speculative-decoding source + in the model's default config must reach CreateModel even though the + customer never touches accept_eula.""" + self._build("openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge") + container_def = self._create_model_container_def() self.assertEqual( - builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] + container_def["AdditionalModelDataSources"], [self.UNGATED_EAGLE_SOURCE] ) - def test_gpt_oss_eagle_source_unaffected_by_accept_eula(self): - """Setting accept_eula=True for an ungated source is harmless: no - ModelAccessConfig is added to it.""" - builder = self._model_builder( - "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" + def test_ungated_additional_source_gets_no_model_access_config(self): + """accept_eula=True must not add a ModelAccessConfig to an ungated + additional source (it applies to the primary model only).""" + self._build( + "openai-reasoning-gpt-oss-20b", accept_eula=True, instance_type="ml.g7e.2xlarge" ) - builder.accept_eula = True - - builder.build() + container_def = self._create_model_container_def() self.assertEqual( - builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] - ) - - # ------------------------------------------------------------------ - # meta-textgeneration-llama-3-1-70b: gated draft_model source in the - # lmi-optimized config (selected via the public set_deployment_config API) - # ------------------------------------------------------------------ - - def test_llama_gated_draft_source_with_accept_eula_builds(self): - """A customer selects the lmi-optimized config and accepts the EULA: - the gated draft model deploys with acceptance recorded in its - ModelAccessConfig.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + container_def["AdditionalModelDataSources"], [self.UNGATED_EAGLE_SOURCE] ) - builder.set_deployment_config( - config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + # The single accept_eula knob still covers the primary (base) model. + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": True}, ) - builder.accept_eula = True - - builder.build() - self.assertEqual( - builder.additional_model_data_sources, - [self.LLAMA_GATED_DRAFT_SOURCE_ACCEPTED], + def test_gated_additional_source_with_accepted_eula_sends_model_access_config(self): + """Accepting the EULA folds ModelAccessConfig into the gated + additional source AND the primary model in the CreateModel request.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=True, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", ) - def test_llama_gated_draft_source_without_accept_eula_raises(self): - """A customer selects the gated draft config but never accepts the - EULA: build() fails with the actionable error instead of silently - deploying the gated channel.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], [self.GATED_DRAFT_SOURCE_ACCEPTED] ) - builder.set_deployment_config( - config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": True}, ) + def test_gated_additional_source_without_eula_raises_and_never_calls_create_model(self): + """No EULA acceptance: build() fails with the actionable error and no + CreateModel call is ever made.""" with self.assertRaises(ValueError) as ctx: - builder.build() + self._build( + "meta-textgeneration-llama-3-1-70b", + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) self.assertIn("accept_eula must be set to True", str(ctx.exception)) + self.session.create_model.assert_not_called() - def test_llama_gated_draft_source_with_accept_eula_false_raises(self): - """A customer explicitly sets accept_eula=False: build() fails the - same way.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" - ) - builder.set_deployment_config( - config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" - ) - builder.accept_eula = False - + def test_gated_additional_source_with_rejected_eula_raises_and_never_calls_create_model(self): + """Explicit accept_eula=False fails the same way.""" with self.assertRaises(ValueError) as ctx: - builder.build() + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=False, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) self.assertIn("accept_eula must be set to True", str(ctx.exception)) - - def test_llama_default_config_has_no_additional_sources(self): - """The same gated model on its default (lmi) config carries no - additional sources: builds without accept_eula, nothing propagated.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + self.session.create_model.assert_not_called() + + def test_unselected_config_sources_do_not_leak_into_create_model(self): + """Config resolution gates which sources apply: the same gated model on + its default (lmi) config has no additional sources, so the CreateModel + request must not carry the lmi-optimized config's gated draft model.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=True, + instance_type="ml.p4d.24xlarge", ) - builder.accept_eula = True - - builder.build() - self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) + container_def = self._create_model_container_def() + self.assertNotIn("AdditionalModelDataSources", container_def) class TestDeployWrappers(unittest.TestCase): From c8527863285e986a7cbddc2387add0fd0e5156d5 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 18:00:06 +0000 Subject: [PATCH 6/8] test(serve): derive expected sources from the spec fixtures Replace the hand-copied expected-source class constants with a helper that reads the captured spec fixture and applies the expected transformation declaratively (PascalCase key mapping, content-bucket resolution, HostingEulaKey absence, EULA folding). The fixture stays the single source of truth for data values; only the behavior under test remains spelled out in the test, avoiding both duplication and tautological assertions. --- .../servers/test_model_builder_servers.py | 96 +++++++++++++------ 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index cb49abaaf8..4ea1bce4cd 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1058,34 +1058,51 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): ROLE_ARN = "arn:aws:iam::123456789012:role/SageMakerRole" - # The real EAGLE source published in gpt-oss's default (lmi-optimized) - # config, as it must appear in the CreateModel request: content bucket - # injected, PascalCase, no HostingEulaKey (the API rejects it). - UNGATED_EAGLE_SOURCE = { - "ChannelName": "eagle", - "S3DataSource": { - "CompressionType": "None", - "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-cache-prod-us-west-2/lmi-eagle-heads/models/gpt-oss-20b-p-eagle/", - }, - } - - # The real gated llama-3.2-1b draft model published in llama-3-1-70b's - # lmi-optimized config, as it must appear in the CreateModel request: - # private content bucket, EULA acceptance folded into ModelAccessConfig, - # no HostingEulaKey. - GATED_DRAFT_SOURCE_ACCEPTED = { - "ChannelName": "draft_model", - "S3DataSource": { - "CompressionType": "None", - "S3DataType": "S3Prefix", - "S3Uri": ( - "s3://jumpstart-private-cache-prod-us-west-2/meta-textgeneration/" - "meta-textgeneration-llama-3-2-1b/artifacts/inference-prepack/v1.0.0/" - ), - "ModelAccessConfig": {"AcceptEula": True}, - }, - } + # The content buckets the sources must resolve into (the raw specs publish + # bucket-less key prefixes): public sources land in the public content + # bucket, gated sources in the private one. + PUBLIC_CONTENT_BUCKET = "jumpstart-cache-prod-us-west-2" + PRIVATE_CONTENT_BUCKET = "jumpstart-private-cache-prod-us-west-2" + + @classmethod + def _spec_additional_source(cls, model_id, config_name="lmi-optimized"): + """The raw additional data source exactly as published in the captured + spec fixture: snake_case keys, bucket-less s3_uri key prefix, and + hosting_eula_key present when the source is gated.""" + with open(cls.SPEC_DIR / f"{model_id}.json") as f: + spec = json.load(f) + sources = spec["inference_config_components"][config_name][ + "hosting_additional_data_sources" + ]["speculative_decoding"] + assert len(sources) == 1, f"expected one source in {model_id}/{config_name}" + return sources[0] + + @classmethod + def _expected_create_model_source(cls, model_id, bucket, accept_eula=False): + """The shape the CreateModel request must carry for the fixture's + source. Data values (channel name, compression, data type, key prefix) + come from the spec fixture so it stays the single source of truth; the + transformation under test is spelled out declaratively here instead of + reusing production code (which would make the assertion a tautology): + + - snake_case spec keys become the PascalCase API keys + - the bucket-less key prefix resolves into the given content bucket + - HostingEulaKey never appears (the API rejects it) + - EULA acceptance, when given, is folded into + S3DataSource.ModelAccessConfig + """ + raw = cls._spec_additional_source(model_id) + expected = { + "ChannelName": raw["channel_name"], + "S3DataSource": { + "CompressionType": raw["s3_data_source"]["compression_type"], + "S3DataType": raw["s3_data_source"]["s3_data_type"], + "S3Uri": f"s3://{bucket}/{raw['s3_data_source']['s3_uri']}", + }, + } + if accept_eula: + expected["S3DataSource"]["ModelAccessConfig"] = {"AcceptEula": True} + return expected def setUp(self): import sagemaker.serve.model_builder as model_builder_module @@ -1171,7 +1188,12 @@ def test_ungated_additional_source_reaches_create_model_without_eula(self): container_def = self._create_model_container_def() self.assertEqual( - container_def["AdditionalModelDataSources"], [self.UNGATED_EAGLE_SOURCE] + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "openai-reasoning-gpt-oss-20b", bucket=self.PUBLIC_CONTENT_BUCKET + ) + ], ) def test_ungated_additional_source_gets_no_model_access_config(self): @@ -1183,7 +1205,12 @@ def test_ungated_additional_source_gets_no_model_access_config(self): container_def = self._create_model_container_def() self.assertEqual( - container_def["AdditionalModelDataSources"], [self.UNGATED_EAGLE_SOURCE] + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "openai-reasoning-gpt-oss-20b", bucket=self.PUBLIC_CONTENT_BUCKET + ) + ], ) # The single accept_eula knob still covers the primary (base) model. self.assertEqual( @@ -1203,7 +1230,14 @@ def test_gated_additional_source_with_accepted_eula_sends_model_access_config(se container_def = self._create_model_container_def() self.assertEqual( - container_def["AdditionalModelDataSources"], [self.GATED_DRAFT_SOURCE_ACCEPTED] + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "meta-textgeneration-llama-3-1-70b", + bucket=self.PRIVATE_CONTENT_BUCKET, + accept_eula=True, + ) + ], ) self.assertEqual( container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], From 06e06d7354ec70c5509d2b7f482e7e40b1a1d7d9 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 18:00:06 +0000 Subject: [PATCH 7/8] refactor(core): apply accept_eula to additional sources in container_def Move the additional-model-data-source EULA handling out of the JumpStart-specific block in _build_for_jumpstart and into _apply_accept_eula_to_additional_model_data_sources next to container_def, the single place the primary model's accept_eula already becomes ModelAccessConfig. Both the primary model and the additional sources are now driven by the same accept_eula flag in adjacent code, and every container-assembly call site (including the deployment-config path that never passes through _build_for_jumpstart) gets the gated-source enforcement. _build_for_jumpstart is reduced to pure propagation. No behavior change on the JumpStart path: all 63 serve tests pass unchanged, plus 103 core session_helper tests. --- .../sagemaker/core/helper/session_helper.py | 49 ++++++++++++++++++- .../sagemaker/serve/model_builder_servers.py | 39 +++------------ 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/sagemaker-core/src/sagemaker/core/helper/session_helper.py b/sagemaker-core/src/sagemaker/core/helper/session_helper.py index ecdd4b95eb..0043b63a4a 100644 --- a/sagemaker-core/src/sagemaker/core/helper/session_helper.py +++ b/sagemaker-core/src/sagemaker/core/helper/session_helper.py @@ -3072,6 +3072,47 @@ def _optimization_job_status(sagemaker_client, job_name): return desc +def _apply_accept_eula_to_additional_model_data_sources(additional_model_data_sources, accept_eula): + """Apply EULA acceptance to additional model data sources. + + This is the additional-source counterpart of the primary model handling in + ``container_def`` below, driven by the same ``accept_eula`` flag. Sources + resolved from JumpStart specs may carry a ``HostingEulaKey``: a truthy value + marks a gated source that requires explicit license acceptance. Acceptance + is folded into the source's ``S3DataSource.ModelAccessConfig`` and a gated + source without acceptance raises before any request is made. The + ``HostingEulaKey`` itself is JumpStart-internal metadata the CreateModel + API rejects, so it is always removed. + + Args: + additional_model_data_sources (list[dict]): AdditionalModelDataSources + entries, possibly carrying JumpStart ``HostingEulaKey`` metadata. + accept_eula (bool): Whether model terms of use have been accepted. + + Returns: + list[dict]: The sources in CreateModel API shape. + + Raises: + ValueError: If a gated source is present and ``accept_eula`` is not True. + """ + applied_sources = [] + for source in additional_model_data_sources: + applied_source = dict(source) + if applied_source.pop("HostingEulaKey", None): + if not accept_eula: + raise ValueError( + "accept_eula must be set to True to deploy this model. " + "Please set accept_eula=True on the ModelBuilder instance " + "to confirm you have read and accepted the end-user license " + "agreement for this model." + ) + s3_data_source = dict(applied_source.get("S3DataSource", {})) + s3_data_source["ModelAccessConfig"] = {"AcceptEula": True} + applied_source["S3DataSource"] = s3_data_source + applied_sources.append(applied_source) + return applied_sources + + def container_def( image_uri, model_data_url=None, @@ -3115,7 +3156,13 @@ def container_def( c_def = {"Image": image_uri, "Environment": env} if additional_model_data_sources: - c_def["AdditionalModelDataSources"] = additional_model_data_sources + c_def["AdditionalModelDataSources"] = ( + _apply_accept_eula_to_additional_model_data_sources( + additional_model_data_sources, accept_eula + ) + if isinstance(additional_model_data_sources, list) + else additional_model_data_sources + ) if isinstance(model_data_url, str) and ( not (model_data_url.startswith("s3://") and model_data_url.endswith("tar.gz")) diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index 57f43d7504..187790d421 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1017,41 +1017,18 @@ def _build_for_jumpstart(self) -> Model: # Propagate additional model data sources resolved from the JumpStart # spec (e.g. speculative decoding draft models like EAGLE). The JumpStart # factory (_add_additional_model_data_sources_to_kwargs) already returns - # these in CreateModel API shape via camel_case_to_pascal_case(...). - # - # A source carrying a truthy ``HostingEulaKey`` is gated and requires the - # user to accept its end-user license agreement, exactly like the base - # model. ``container_def`` folds ``accept_eula`` into the base model's - # ModelAccessConfig but passes additional sources through untouched, so - # the acceptance must be translated onto each gated source here: enforce - # ``accept_eula`` and move it into the source's ``ModelAccessConfig`` - # (mirroring _add_model_access_configs_to_model_data_sources and the base - # model's handling in _create_model). Ungated sources simply drop the - # JumpStart-internal ``HostingEulaKey`` metadata, which the CreateModel - # API rejects. Without any of this, sources declared in the spec are - # dropped from the CreateModel call and the container fails to find the - # referenced artifacts at runtime. + # these in CreateModel API shape via camel_case_to_pascal_case(...); + # EULA handling (enforcing accept_eula on gated sources and stripping + # the JumpStart-internal HostingEulaKey) happens in ``container_def``, + # the same place the primary model's accept_eula is applied. Without + # this propagation, sources declared in the spec are dropped from the + # CreateModel call and the container fails to find the referenced + # artifacts at runtime. additional_model_data_sources = getattr( init_kwargs, "additional_model_data_sources", None ) if isinstance(additional_model_data_sources, list) and additional_model_data_sources: - accept_eula = getattr(self, "accept_eula", None) - sanitized_sources = [] - for source in additional_model_data_sources: - sanitized_source = dict(source) - if sanitized_source.pop("HostingEulaKey", None): - if not accept_eula: - raise ValueError( - "accept_eula must be set to True to deploy this model. " - "Please set accept_eula=True on the ModelBuilder instance " - "to confirm you have read and accepted the end-user license " - "agreement for this model." - ) - s3_data_source = dict(sanitized_source.get("S3DataSource", {})) - s3_data_source["ModelAccessConfig"] = {"AcceptEula": True} - sanitized_source["S3DataSource"] = s3_data_source - sanitized_sources.append(sanitized_source) - self.additional_model_data_sources = sanitized_sources + self.additional_model_data_sources = additional_model_data_sources # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: From 734a74b547c29f08fc78bfba08b20cc406d0db82 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 19:37:58 +0000 Subject: [PATCH 8/8] refactor(serve): inline additional-source EULA fold in _build_for_jumpstart Scope the EULA handling down to mirror the primary model exactly: strip the JumpStart-internal HostingEulaKey (not part of the CreateModel API shape) and fold accept_eula into each source's ModelAccessConfig when set. No client-side gatedness check or raise: the control plane determines gatedness from the artifact's bucket and rejects CreateModel with its own EULA validation error, for the primary model and additional sources alike. This reverts the sagemaker-core half of the previous commit: session_helper.py returns to upstream and the whole change lives in the JumpStart builder that produces the spec-derived sources. Tests assert the new contract on the CreateModel request: acceptance folded when accept_eula is set (True or False), no ModelAccessConfig when unset. --- .../sagemaker/core/helper/session_helper.py | 49 +--------- .../sagemaker/serve/model_builder_servers.py | 33 +++++-- .../servers/test_model_builder_servers.py | 97 ++++++++++++------- 3 files changed, 88 insertions(+), 91 deletions(-) diff --git a/sagemaker-core/src/sagemaker/core/helper/session_helper.py b/sagemaker-core/src/sagemaker/core/helper/session_helper.py index 0043b63a4a..ecdd4b95eb 100644 --- a/sagemaker-core/src/sagemaker/core/helper/session_helper.py +++ b/sagemaker-core/src/sagemaker/core/helper/session_helper.py @@ -3072,47 +3072,6 @@ def _optimization_job_status(sagemaker_client, job_name): return desc -def _apply_accept_eula_to_additional_model_data_sources(additional_model_data_sources, accept_eula): - """Apply EULA acceptance to additional model data sources. - - This is the additional-source counterpart of the primary model handling in - ``container_def`` below, driven by the same ``accept_eula`` flag. Sources - resolved from JumpStart specs may carry a ``HostingEulaKey``: a truthy value - marks a gated source that requires explicit license acceptance. Acceptance - is folded into the source's ``S3DataSource.ModelAccessConfig`` and a gated - source without acceptance raises before any request is made. The - ``HostingEulaKey`` itself is JumpStart-internal metadata the CreateModel - API rejects, so it is always removed. - - Args: - additional_model_data_sources (list[dict]): AdditionalModelDataSources - entries, possibly carrying JumpStart ``HostingEulaKey`` metadata. - accept_eula (bool): Whether model terms of use have been accepted. - - Returns: - list[dict]: The sources in CreateModel API shape. - - Raises: - ValueError: If a gated source is present and ``accept_eula`` is not True. - """ - applied_sources = [] - for source in additional_model_data_sources: - applied_source = dict(source) - if applied_source.pop("HostingEulaKey", None): - if not accept_eula: - raise ValueError( - "accept_eula must be set to True to deploy this model. " - "Please set accept_eula=True on the ModelBuilder instance " - "to confirm you have read and accepted the end-user license " - "agreement for this model." - ) - s3_data_source = dict(applied_source.get("S3DataSource", {})) - s3_data_source["ModelAccessConfig"] = {"AcceptEula": True} - applied_source["S3DataSource"] = s3_data_source - applied_sources.append(applied_source) - return applied_sources - - def container_def( image_uri, model_data_url=None, @@ -3156,13 +3115,7 @@ def container_def( c_def = {"Image": image_uri, "Environment": env} if additional_model_data_sources: - c_def["AdditionalModelDataSources"] = ( - _apply_accept_eula_to_additional_model_data_sources( - additional_model_data_sources, accept_eula - ) - if isinstance(additional_model_data_sources, list) - else additional_model_data_sources - ) + c_def["AdditionalModelDataSources"] = additional_model_data_sources if isinstance(model_data_url, str) and ( not (model_data_url.startswith("s3://") and model_data_url.endswith("tar.gz")) diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index 187790d421..14c9dc19fe 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1017,18 +1017,35 @@ def _build_for_jumpstart(self) -> Model: # Propagate additional model data sources resolved from the JumpStart # spec (e.g. speculative decoding draft models like EAGLE). The JumpStart # factory (_add_additional_model_data_sources_to_kwargs) already returns - # these in CreateModel API shape via camel_case_to_pascal_case(...); - # EULA handling (enforcing accept_eula on gated sources and stripping - # the JumpStart-internal HostingEulaKey) happens in ``container_def``, - # the same place the primary model's accept_eula is applied. Without - # this propagation, sources declared in the spec are dropped from the - # CreateModel call and the container fails to find the referenced - # artifacts at runtime. + # these in CreateModel API shape via camel_case_to_pascal_case(...), + # except for two adjustments applied here, mirroring how ``container_def`` + # treats the primary model's ``ModelDataSource``: + # - the JumpStart-internal ``HostingEulaKey`` is removed (it is not part + # of the CreateModel API shape, request validation rejects it), and + # - when ``accept_eula`` is set, it is folded into each source's + # ``S3DataSource.ModelAccessConfig``. + # Gated-source EULA enforcement is server-side, same as for the primary + # model: the control plane determines gatedness from the artifact's + # bucket and rejects CreateModel with an EULA validation error when a + # gated source lacks ``ModelAccessConfig`` with ``AcceptEula`` true. + # Without this propagation, sources declared in the spec are dropped + # from the CreateModel call and the container fails to find the + # referenced artifacts at runtime. additional_model_data_sources = getattr( init_kwargs, "additional_model_data_sources", None ) if isinstance(additional_model_data_sources, list) and additional_model_data_sources: - self.additional_model_data_sources = additional_model_data_sources + accept_eula = getattr(self, "accept_eula", None) + prepared_sources = [] + for source in additional_model_data_sources: + prepared_source = dict(source) + prepared_source.pop("HostingEulaKey", None) + if accept_eula is not None: + s3_data_source = dict(prepared_source.get("S3DataSource", {})) + s3_data_source["ModelAccessConfig"] = {"AcceptEula": accept_eula} + prepared_source["S3DataSource"] = s3_data_source + prepared_sources.append(prepared_source) + self.additional_model_data_sources = prepared_sources # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 4ea1bce4cd..1ef21e63da 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1050,8 +1050,9 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): Everything in between runs for real: JumpStart model-id detection, JumpStartModelSpecs parsing, inference-config resolution, the get_init_kwargs factory (image URI resolution, content-bucket injection, - PascalCase shaping), _build_for_jumpstart's EULA translation, and - _prepare_container_def_base assembling the CreateModel request. + PascalCase shaping), the propagation in _build_for_jumpstart, the + accept_eula application in container_def (shared with the primary model), + and _prepare_container_def_base assembling the CreateModel request. """ SPEC_DIR = Path(__file__).parent / "data" / "jumpstart_specs" @@ -1078,7 +1079,7 @@ def _spec_additional_source(cls, model_id, config_name="lmi-optimized"): return sources[0] @classmethod - def _expected_create_model_source(cls, model_id, bucket, accept_eula=False): + def _expected_create_model_source(cls, model_id, bucket, accept_eula=None): """The shape the CreateModel request must carry for the fixture's source. Data values (channel name, compression, data type, key prefix) come from the spec fixture so it stays the single source of truth; the @@ -1088,8 +1089,8 @@ def _expected_create_model_source(cls, model_id, bucket, accept_eula=False): - snake_case spec keys become the PascalCase API keys - the bucket-less key prefix resolves into the given content bucket - HostingEulaKey never appears (the API rejects it) - - EULA acceptance, when given, is folded into - S3DataSource.ModelAccessConfig + - accept_eula, when set, is folded into S3DataSource.ModelAccessConfig + exactly like the primary model's ModelDataSource """ raw = cls._spec_additional_source(model_id) expected = { @@ -1100,8 +1101,8 @@ def _expected_create_model_source(cls, model_id, bucket, accept_eula=False): "S3Uri": f"s3://{bucket}/{raw['s3_data_source']['s3_uri']}", }, } - if accept_eula: - expected["S3DataSource"]["ModelAccessConfig"] = {"AcceptEula": True} + if accept_eula is not None: + expected["S3DataSource"]["ModelAccessConfig"] = {"AcceptEula": accept_eula} return expected def setUp(self): @@ -1196,9 +1197,12 @@ def test_ungated_additional_source_reaches_create_model_without_eula(self): ], ) - def test_ungated_additional_source_gets_no_model_access_config(self): - """accept_eula=True must not add a ModelAccessConfig to an ungated - additional source (it applies to the primary model only).""" + def test_accept_eula_applies_to_every_model_data_source_uniformly(self): + """The single accept_eula knob folds the same ModelAccessConfig into + the primary model AND each additional source, exactly like the primary + model's ModelDataSource handling. Gatedness is not the SDK's call: the + service determines it from the artifact's bucket and ignores the config + on ungated sources.""" self._build( "openai-reasoning-gpt-oss-20b", accept_eula=True, instance_type="ml.g7e.2xlarge" ) @@ -1208,11 +1212,12 @@ def test_ungated_additional_source_gets_no_model_access_config(self): container_def["AdditionalModelDataSources"], [ self._expected_create_model_source( - "openai-reasoning-gpt-oss-20b", bucket=self.PUBLIC_CONTENT_BUCKET + "openai-reasoning-gpt-oss-20b", + bucket=self.PUBLIC_CONTENT_BUCKET, + accept_eula=True, ) ], ) - # The single accept_eula knob still covers the primary (base) model. self.assertEqual( container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], {"AcceptEula": True}, @@ -1244,31 +1249,53 @@ def test_gated_additional_source_with_accepted_eula_sends_model_access_config(se {"AcceptEula": True}, ) - def test_gated_additional_source_without_eula_raises_and_never_calls_create_model(self): - """No EULA acceptance: build() fails with the actionable error and no - CreateModel call is ever made.""" - with self.assertRaises(ValueError) as ctx: - self._build( - "meta-textgeneration-llama-3-1-70b", - inference_config_name="lmi-optimized", - instance_type="ml.p4d.24xlarge", - ) + def test_gated_additional_source_without_eula_sends_no_model_access_config(self): + """No EULA decision: the request carries the gated source without any + ModelAccessConfig. Enforcement is the service's: the SageMaker control + plane resolves the private-bucket URI as gated and rejects CreateModel + with an EULA validation error, the same as for a gated primary model.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) - self.assertIn("accept_eula must be set to True", str(ctx.exception)) - self.session.create_model.assert_not_called() + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "meta-textgeneration-llama-3-1-70b", bucket=self.PRIVATE_CONTENT_BUCKET + ) + ], + ) - def test_gated_additional_source_with_rejected_eula_raises_and_never_calls_create_model(self): - """Explicit accept_eula=False fails the same way.""" - with self.assertRaises(ValueError) as ctx: - self._build( - "meta-textgeneration-llama-3-1-70b", - accept_eula=False, - inference_config_name="lmi-optimized", - instance_type="ml.p4d.24xlarge", - ) - - self.assertIn("accept_eula must be set to True", str(ctx.exception)) - self.session.create_model.assert_not_called() + def test_gated_additional_source_with_rejected_eula_sends_acceptance_false(self): + """Explicit accept_eula=False is transmitted faithfully as + ModelAccessConfig={"AcceptEula": False} on every source, exactly like + the primary model. The service rejects the gated sources.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=False, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) + + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "meta-textgeneration-llama-3-1-70b", + bucket=self.PRIVATE_CONTENT_BUCKET, + accept_eula=False, + ) + ], + ) + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": False}, + ) def test_unselected_config_sources_do_not_leak_into_create_model(self): """Config resolution gates which sources apply: the same gated model on