diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index 08a77e0d1e..14c9dc19fe 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1014,6 +1014,39 @@ def _build_for_jumpstart(self) -> Model: if getattr(init_kwargs, "model_reference_arn", None): self.model_reference_arn = init_kwargs.model_reference_arn + # Propagate additional model data sources resolved from the JumpStart + # spec (e.g. speculative decoding draft models like EAGLE). The JumpStart + # factory (_add_additional_model_data_sources_to_kwargs) already returns + # these in CreateModel API shape via camel_case_to_pascal_case(...), + # except for two adjustments applied here, mirroring how ``container_def`` + # treats the primary model's ``ModelDataSource``: + # - the JumpStart-internal ``HostingEulaKey`` is removed (it is not part + # of the CreateModel API shape, request validation rejects it), and + # - when ``accept_eula`` is set, it is folded into each source's + # ``S3DataSource.ModelAccessConfig``. + # Gated-source EULA enforcement is server-side, same as for the primary + # model: the control plane determines gatedness from the artifact's + # bucket and rejects CreateModel with an EULA validation error when a + # gated source lacks ``ModelAccessConfig`` with ``AcceptEula`` true. + # Without this propagation, sources declared in the spec are dropped + # from the CreateModel call and the container fails to find the + # referenced artifacts at runtime. + additional_model_data_sources = getattr( + init_kwargs, "additional_model_data_sources", None + ) + if isinstance(additional_model_data_sources, list) and additional_model_data_sources: + accept_eula = getattr(self, "accept_eula", None) + prepared_sources = [] + for source in additional_model_data_sources: + prepared_source = dict(source) + prepared_source.pop("HostingEulaKey", None) + if accept_eula is not None: + s3_data_source = dict(prepared_source.get("S3DataSource", {})) + s3_data_source["ModelAccessConfig"] = {"AcceptEula": accept_eula} + prepared_source["S3DataSource"] = s3_data_source + prepared_sources.append(prepared_source) + self.additional_model_data_sources = prepared_sources + # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: if ( diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json new file mode 100644 index 0000000000..486ca44a27 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json @@ -0,0 +1,38 @@ +[ + { + "model_id": "pytorch-ic-mobilenet-v2", + "version": "3.0.25", + "min_version": "2.189.0", + "spec_key": "community_models/pytorch-ic-mobilenet-v2/specs_v3.0.25.json", + "deprecated": false, + "provider": "pytorch", + "search_keywords": [ + "MobileNet V2", + "Vision", + "ResNet 18" + ] + }, + { + "model_id": "openai-reasoning-gpt-oss-20b", + "version": "3.38.0", + "min_version": "2.244.2", + "spec_key": "community_models/openai-reasoning-gpt-oss-20b/specs_v3.38.0.json", + "deprecated": false, + "provider": "openai", + "search_keywords": [ + "Text Generation", + "Reasoning" + ] + }, + { + "model_id": "meta-textgeneration-llama-3-1-70b", + "version": "2.16.2", + "min_version": "2.232.1", + "spec_key": "community_models/meta-textgeneration-llama-3-1-70b/specs_v2.16.2.json", + "deprecated": false, + "provider": "meta", + "search_keywords": [ + "Text Generation" + ] + } +] \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json new file mode 100644 index 0000000000..3b53a2e449 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json @@ -0,0 +1,1910 @@ +{ + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_training_dataset_key": "training-datasets/sec_amazon/", + "default_training_instance_type": "ml.g5.48xlarge", + "deprecated": false, + "dynamic_container_deployment_supported": true, + "estimator_kwargs": { + "disable_output_compression": true, + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fine_tuning_supported": true, + "fit_kwargs": {}, + "gated_bucket": true, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_eula_key": "fmhMetadata/eula/llama3_1Eula.txt", + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "huggingface_id": "meta-llama/Llama-3.1-70B", + "hyperparameters": [ + { + "default": 5, + "max": 1000, + "min": 1, + "name": "epoch", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.0001, + "max": 1, + "min": 1e-08, + "name": "learning_rate", + "scope": "algorithm", + "type": "float" + }, + { + "default": 8, + "min": 1, + "name": "lora_r", + "scope": "algorithm", + "type": "int" + }, + { + "default": 32, + "min": 1, + "name": "lora_alpha", + "scope": "algorithm", + "type": "int" + }, + { + "default": "q_proj,v_proj", + "name": "target_modules", + "scope": "algorithm", + "type": "text" + }, + { + "default": 0.05, + "max": 1, + "min": 0, + "name": "lora_dropout", + "scope": "algorithm", + "type": "float" + }, + { + "default": "False", + "name": "instruction_tuned", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "False", + "name": "chat_dataset", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "True", + "name": "add_input_output_demarcation_key", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": 1, + "max": 1000, + "min": 1, + "name": "per_device_train_batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": 1, + "max": 1000, + "min": 1, + "name": "per_device_eval_batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_train_samples", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_val_samples", + "scope": "algorithm", + "type": "int" + }, + { + "default": 10, + "max": 1000, + "min": 1, + "name": "seed", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_input_length", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.2, + "max": 1, + "min": 0, + "name": "validation_split_ratio", + "scope": "algorithm", + "type": "float" + }, + { + "default": 0, + "min": 0, + "name": "train_data_split_seed", + "scope": "algorithm", + "type": "int" + }, + { + "default": "None", + "name": "preprocessing_num_workers", + "scope": "algorithm", + "type": "text" + }, + { + "default": "True", + "name": "int8_quantization", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "False", + "name": "enable_fsdp", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "Llama3.1", + "name": "chat_template", + "scope": "algorithm", + "type": "text" + }, + { + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "name": "sagemaker_submit_directory", + "scope": "container", + "type": "text" + }, + { + "default": "transfer_learning.py", + "name": "sagemaker_program", + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "sagemaker_container_log_level", + "scope": "container", + "type": "text" + } + ], + "incremental_training_supported": false, + "inference_config_components": { + "lmi": { + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_payloads": { + "englishToFrench": { + "body": { + "inputs": "Translate English to French:\nsea otter => loutre de mer\npeppermint => menthe poivr\u00e9e\nplush girafe => girafe peluche\ncheese =>", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "meaningOfLife": { + "body": { + "inputs": "I believe the meaning of life is", + "parameters": { + "details": true, + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "teamMessage": { + "body": { + "inputs": "A brief message congratulating the team on the launch:\n\nHi everyone,\n\nI just ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "theoryOfRelativity": { + "body": { + "inputs": "Simply put, the theory of relativity states that ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.p4d.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + } + } + }, + "ml.p4de.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + }, + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "true", + "name": "OPTION_ENFORCE_EAGER", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge", + "ml.g6.48xlarge", + "ml.p5.48xlarge" + ] + }, + "lmi-optimized": { + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_payloads": { + "englishToFrench": { + "body": { + "inputs": "Translate English to French:\nsea otter => loutre de mer\npeppermint => menthe poivr\u00e9e\nplush girafe => girafe peluche\ncheese =>", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "meaningOfLife": { + "body": { + "inputs": "I believe the meaning of life is", + "parameters": { + "details": true, + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "teamMessage": { + "body": { + "inputs": "A brief message congratulating the team on the launch:\n\nHi everyone,\n\nI just ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "theoryOfRelativity": { + "body": { + "inputs": "Simply put, the theory of relativity states that ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "artifact_version": "v1.0.0", + "channel_name": "draft_model", + "hosting_eula_key": "fmhMetadata/eula/llama3_2Eula.txt", + "provider": { + "classification": "gated", + "name": "JumpStart" + }, + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "meta-textgeneration/meta-textgeneration-llama-3-2-1b/artifacts/inference-prepack/v1.0.0/" + } + } + ] + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.p4d.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + } + } + }, + "ml.p4de.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + }, + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/additional-model-data-sources/draft_model", + "name": "OPTION_SPECULATIVE_DRAFT_MODEL", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.g6.48xlarge", + "ml.p4d.24xlarge", + "ml.p5.48xlarge" + ] + } + }, + "inference_config_rankings": { + "overall": { + "description": "default", + "rankings": [ + "lmi", + "lmi-optimized" + ] + } + }, + "inference_configs": { + "lmi": { + "benchmark_metrics": { + "ml.g5.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "2.68" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "18.6" + } + ], + "ml.g6.48xlarge": [ + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "1.41" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "4.9" + } + ], + "ml.p4d.24xlarge": [ + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "0.11" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "37.4" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "0.13" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "21.6" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "0.16" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "13.9" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "0.20" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "8.4" + }, + { + "concurrency": "128", + "name": "latency", + "unit": "sec", + "value": "3.91" + }, + { + "concurrency": "128", + "name": "throughput", + "unit": "tokens/sec", + "value": "7.6" + } + ], + "ml.p5.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "48.3" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "48.3" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "46.3" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "42.1" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "0.09" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "35.8" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "0.09" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "26.6" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "0.10" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "17.3" + } + ] + }, + "component_names": [ + "lmi" + ] + }, + "lmi-optimized": { + "acceleration_configs": [ + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "TRT-LLM 0.11.0 in LMI v11 does not support llama 3.1" + } + }, + "enabled": false, + "type": "Compilation" + }, + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "LMI v11 does not support Speculative Decoding for TRT" + } + }, + "enabled": true, + "type": "Speculative-Decoding" + }, + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "TRT-LLM 0.11.0 in LMI v11 does not support llama 3.1" + } + }, + "enabled": false, + "type": "Quantization" + } + ], + "benchmark_metrics": { + "ml.g5.48xlarge": [ + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "2.94" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "16.1" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "4.04" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "6.1" + } + ], + "ml.g6.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.93" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "23.5" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "1.14" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "17.7" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "1.15" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "11.2" + } + ], + "ml.p4d.24xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.10" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "89.4" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "0.13" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "81.2" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "0.15" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "49.1" + } + ] + }, + "component_names": [ + "lmi-optimized" + ] + } + }, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "true", + "name": "OPTION_ENFORCE_EAGER", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "input_modalities": [ + "Text" + ], + "languages": [ + "English", + "German", + "French", + "Italian", + "Portuguese", + "Hindi", + "Spanish", + "Thai" + ], + "license": "Llama 3.1 Community License", + "metrics": [ + { + "Name": "huggingface-textgeneration:eval-loss", + "Regex": "eval_epoch_loss=tensor\\(([0-9\\.]+)" + }, + { + "Name": "huggingface-textgeneration:eval-ppl", + "Regex": "eval_ppl=tensor\\(([0-9\\.]+)" + }, + { + "Name": "huggingface-textgeneration:train-loss", + "Regex": "train_epoch_loss=([0-9\\.]+)" + } + ], + "min_sdk_version": "2.232.1", + "ml_framework": "Meta", + "model_access": "Gated", + "model_id": "meta-textgeneration-llama-3-1-70b", + "model_kwargs": {}, + "model_size": "70B-100B", + "output_modalities": [ + "Text" + ], + "provider": "meta", + "resource_name_base": "meta-textgeneration-llama-3-1-70b", + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge", + "ml.g6.48xlarge", + "ml.p5.48xlarge" + ], + "supported_training_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge" + ], + "training_artifact_key": "meta-training/train-meta-textgeneration-llama-3-1-70b.tar.gz", + "training_dependencies": [ + "accelerate==0.33.0", + "bitsandbytes==0.39.1", + "black==23.7.0", + "brotli==1.0.9", + "datasets==2.14.1", + "docstring-parser==0.16", + "fire==0.5.0", + "huggingface-hub==0.24.2", + "inflate64==0.3.1", + "loralib==0.1.1", + "multivolumefile==0.2.3", + "mypy-extensions==1.0.0", + "nvidia-cublas-cu12==12.1.3.1", + "nvidia-cuda-cupti-cu12==12.1.105", + "nvidia-cuda-nvrtc-cu12==12.1.105", + "nvidia-cuda-runtime-cu12==12.1.105", + "nvidia-cudnn-cu12==8.9.2.26", + "nvidia-cufft-cu12==11.0.2.54", + "nvidia-curand-cu12==10.3.2.106", + "nvidia-cusolver-cu12==11.4.5.107", + "nvidia-cusolver-cu12==11.4.5.107", + "nvidia-cusparse-cu12==12.1.0.106", + "nvidia-nccl-cu12==2.19.3", + "nvidia-nvjitlink-cu12==12.3.101", + "nvidia-nvtx-cu12==12.1.105", + "pathspec==0.11.1", + "peft==0.4.0", + "py7zr==0.20.5", + "pybcj==1.0.1", + "pycryptodomex==3.18.0", + "pyppmd==1.0.0", + "pyzstd==0.15.9", + "safetensors==0.4.2", + "sagemaker_jumpstart_huggingface_script_utilities==1.2.7", + "sagemaker_jumpstart_script_utilities==1.1.9", + "scipy==1.11.1", + "shtab==1.7.1", + "termcolor==2.3.0", + "texttable==1.6.7", + "tokenize-rt==5.1.0", + "tokenizers==0.19.1", + "torch==2.2.0", + "transformers==4.43.1", + "triton==2.2.0", + "trl==0.8.1", + "typing-extensions==4.8.0", + "tyro==0.7.3" + ], + "training_ecr_specs": { + "framework": "huggingface", + "framework_version": "2.0.0", + "huggingface_transformers_version": "4.28.1", + "py_version": "py310" + }, + "training_enable_network_isolation": true, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "gpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-east-1": { + "gpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-3": { + "gpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-south-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-3": { + "gpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ca-central-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ca-west-1": { + "gpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "cn-north-1": { + "gpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "cn-northwest-1": { + "gpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-central-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-north-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-south-1": { + "gpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-3": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "il-central-1": { + "gpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "me-central-1": { + "gpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "me-south-1": { + "gpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "sa-east-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-east-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-east-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-gov-east-1": { + "gpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-gov-west-1": { + "gpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-iso-east-1": { + "gpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-isob-east-1": { + "gpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-west-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-west-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + } + }, + "variants": { + "g4dn": { + "properties": { + "gated_model_key_env_var_value": "meta-training/g4dn/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g5": { + "properties": { + "gated_model_key_env_var_value": "meta-training/g5/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p3dn": { + "properties": { + "gated_model_key_env_var_value": "meta-training/p3dn/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p4d": { + "properties": { + "gated_model_key_env_var_value": "meta-training/p4d/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + } + } + }, + "training_prepacked_script_key": "source-directory-tarballs/training/meta-textgeneration/prepack/inference-meta-textgeneration/v1.2.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.2.0", + "training_script_key": "source-directory-tarballs/training/meta-textgeneration/v1.2.0/sourcedir.tar.gz", + "training_supported": true, + "training_volume_size": 256, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://ai.meta.com/resources/models-and-libraries/llama-downloads/", + "validation_supported": true, + "version": "2.16.2" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json new file mode 100644 index 0000000000..b113d8bdff --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json @@ -0,0 +1,3801 @@ +{ + "context_window": "32K-128K", + "default_inference_instance_type": "ml.g7e.2xlarge", + "deprecated": false, + "dynamic_container_deployment_supported": true, + "fine_tuning_supported": false, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "vllm", + "framework_version": "0.18.0", + "py_version": "py312" + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "huggingface_id": "openai/gpt-oss-20b", + "incremental_training_supported": false, + "inference_config_components": { + "lmi": { + "default_inference_instance_type": "ml.g7e.2xlarge", + "default_payloads": { + "marketing": { + "body": { + "max_tokens": 200, + "messages": [ + { + "content": "Olly has just promoted you to the marketing department.", + "role": "system" + }, + { + "content": "Create ad text promoting Olly supplements for woman over 40 years using the following keywords: mental, health, sleep, energy, mood, cognitive, adult, children, supplement, and support. Include a slogan.", + "role": "user" + } + ], + "temperature": 0.7, + "top_p": 1 + }, + "content_type": "application/json" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.34.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-east-2": { + "alias_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-6": { + "alias_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-isoe-west-1": { + "alias_ecr_uri_1": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eusc-de-east-1": { + "alias_ecr_uri_1": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g7e.2xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "1" + }, + "resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5e.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5en.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + }, + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ] + }, + "lmi-optimized": { + "default_inference_instance_type": "ml.g7e.2xlarge", + "default_payloads": { + "marketing": { + "body": { + "max_tokens": 200, + "messages": [ + { + "content": "Olly has just promoted you to the marketing department.", + "role": "system" + }, + { + "content": "Create ad text promoting Olly supplements for woman over 40 years using the following keywords: mental, health, sleep, energy, mood, cognitive, adult, children, supplement, and support. Include a slogan.", + "role": "user" + } + ], + "temperature": 0.7, + "top_p": 1 + }, + "content_type": "application/json" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 3600, + "model_data_download_timeout": 3600 + }, + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "artifact_version": "v1.0.0", + "channel_name": "eagle", + "provider": { + "classification": "non-gated", + "name": "JumpStart" + }, + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "lmi-eagle-heads/models/gpt-oss-20b-p-eagle/" + } + } + ] + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "vllm", + "framework_version": "0.18.0", + "py_version": "py312" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-east-2": { + "alias_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-6": { + "alias_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-isoe-west-1": { + "alias_ecr_uri_1": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eusc-de-east-1": { + "alias_ecr_uri_1": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g7e.2xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "1" + }, + "resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5e.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5en.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + } + } + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + }, + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "8192", + "name": "SM_VLLM_MAX_MODEL_LEN", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "{\"model\": \"/opt/ml/additional-model-data-sources/eagle\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3, \"parallel_drafting\": true}", + "name": "SM_VLLM_SPECULATIVE_CONFIG", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SM_VLLM_TENSOR_PARALLEL_SIZE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ] + } + }, + "inference_config_rankings": { + "overall": { + "description": "default", + "rankings": [ + "lmi-optimized", + "lmi" + ] + } + }, + "inference_configs": { + "lmi": { + "component_names": [ + "lmi" + ] + }, + "lmi-optimized": { + "acceleration_configs": [ + { + "enabled": true, + "type": "Speculative-Decoding" + } + ], + "benchmark_metrics": { + "ml.g7e.2xlarge": [ + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "40" + }, + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "50.0" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "80" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "25.0" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "160" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "12.5" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "6.25" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "3.12" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "1.56" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "1.0" + }, + { + "concurrency": "128", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "128", + "name": "latency", + "unit": "sec", + "value": "1.0" + } + ] + }, + "component_names": [ + "lmi-optimized" + ] + } + }, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "8192", + "name": "SM_VLLM_MAX_MODEL_LEN", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "{\"model\": \"/opt/ml/additional-model-data-sources/eagle\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3, \"parallel_drafting\": true}", + "name": "SM_VLLM_SPECULATIVE_CONFIG", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SM_VLLM_TENSOR_PARALLEL_SIZE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "input_modalities": [ + "Text" + ], + "languages": [ + "English" + ], + "license": "Apache 2.0", + "metrics": [], + "min_sdk_version": "2.244.2", + "ml_framework": "PyTorch", + "model_access": "Open Weight", + "model_id": "openai-reasoning-gpt-oss-20b", + "model_kwargs": {}, + "model_size": "10B-70B", + "output_modalities": [ + "Text" + ], + "provider": "openai", + "recipe_collection": [ + { + "customization_technique": "DPO", + "display_name": "GPT OSS 20B Direct Preference Optimization on GPU, 4K sequence length", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_override_params_k8s_v2.2.4.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_payload_template_k8s_v2.2.4.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_dpo", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_dpo.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_override_params_sm_jobs_v2.2.4.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_payload_template_sm_jobs_v2.2.4.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.4" + ] + }, + { + "customization_technique": "SFT", + "display_name": "GPT OSS 20B Supervised Fine-Tuning with LoRA on GPU, 4K sequence length", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_override_params_k8s_v2.3.1.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_payload_template_k8s_v2.3.1.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_sft_lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_sft_lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_override_params_sm_jobs_v2.3.1.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_payload_template_sm_jobs_v2.3.1.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.3.1" + ] + }, + { + "customization_technique": "RLAIF", + "display_name": "GPT OSS 20B GRPO RLAIF Fine-Tuning with LoRA", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_override_params_k8s_v2.2.3.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_payload_template_k8s_v2.2.3.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "verl-grpo-rlaif-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlaif-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_override_params_sm_jobs_v2.2.3.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_payload_template_sm_jobs_v2.2.3.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.3" + ] + }, + { + "customization_technique": "RLVR", + "display_name": "GPT OSS 20B GRPO RLVR Fine-Tuning with LoRA", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_override_params_k8s_v2.2.2.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_payload_template_k8s_v2.2.2.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "verl-grpo-rlvr-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlvr-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "1K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_override_params_sm_jobs_v2.2.2.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_payload_template_sm_jobs_v2.2.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.2" + ] + }, + { + "display_name": "Open Source Evaluation Evaluation on GPU - Openai Reasoning Gpt Oss 20B", + "evaluation_type": "DeterministicEvaluation", + "hardware": "GPU", + "hp_eks_override_params_s3_key": "recipes/openai_reasoning_gpt_oss_20b_override_params_k8s_v2.0.2.json", + "hp_eks_payload_template_s3_key": "recipes/openai_reasoning_gpt_oss_20b_payload_template_k8s_v2.0.2.yaml", + "name": "open-source-eval-openai-reasoning-gpt-oss-20b-deterministic", + "regional_parameters": { + "gamma": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + }, + "prod": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + } + }, + "smtj_override_params_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-deterministic_override_params_sm_jobs_v2.0.2.json", + "smtj_recipe_template_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-deterministic_payload_template_sm_jobs_v2.0.2.yaml", + "supported_instance_types": [ + "ml.g5.12xlarge", + "ml.g5.24xlarge", + "ml.g5.48xlarge", + "ml.g6.24xlarge", + "ml.g6.48xlarge", + "ml.p4d.24xlarge", + "ml.p4de.24xlarge" + ], + "type": "Evaluation", + "versions": [ + "2.0.1" + ] + }, + { + "display_name": "Open Source Evaluation Evaluation on GPU - Openai Reasoning Gpt Oss 20B", + "evaluation_type": "LLMAJEvaluation", + "hardware": "CPU", + "hp_eks_override_params_s3_key": "recipes/openai_reasoning_gpt_oss_20b_override_params_k8s_v2.0.2.json", + "hp_eks_payload_template_s3_key": "recipes/openai_reasoning_gpt_oss_20b_payload_template_k8s_v2.0.2.yaml", + "name": "open-source-eval-openai-reasoning-gpt-oss-20b-llmaj", + "regional_parameters": { + "gamma": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + }, + "prod": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + } + }, + "smtj_override_params_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-llmaj_override_params_sm_jobs_v2.0.2.json", + "smtj_recipe_template_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-llmaj_payload_template_sm_jobs_v2.0.2.yaml", + "supported_instance_types": [ + "ml.t3.large" + ], + "type": "Evaluation", + "versions": [ + "2.0.1" + ] + }, + { + "customization_technique": "MTRL", + "display_name": "Agentic RFT for GPT OSS 20B Lora", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "name": "mtrl-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/mtrl-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-1": { + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-south-1": { + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-1": { + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-2": { + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-central-1": { + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-north-1": { + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-south-2": { + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-1": { + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-2": { + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "sa-east-1": { + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-east-1": { + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-east-2": { + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-west-1": { + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + } + }, + "sequence_length": "64K", + "smtj_override_params_s3_key": "recipes/mtrl-gpt-oss-20b-lora_override_params_sm_jobs_v1.0.22.json", + "smtj_recipe_template_s3_key": "recipes/mtrl-gpt-oss-20b-lora_payload_template_sm_jobs_v1.0.22.yaml", + "type": "FineTuning", + "versions": [ + "1.0.22" + ] + }, + { + "display_name": "MTRL Evaluation", + "evaluation_type": "MTRLEvaluation", + "name": "mtrl-eval-openai-reasoning-gpt-oss-20b", + "recipe_file_path": "recipes/evaluation/mtrl/mtrl_eval.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "prod": { + "us-east-1": { + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + } + }, + "smtj_override_params_s3_key": "recipes/mtrl-eval-openai-reasoning-gpt-oss-20b_override_params_sm_jobs_v1.0.12.json", + "smtj_recipe_template_s3_key": "recipes/mtrl-eval-openai-reasoning-gpt-oss-20b_payload_template_sm_jobs_v1.0.12.yaml", + "type": "Evaluation", + "versions": [ + "1.0.12" + ] + }, + { + "customization_technique": "SFT", + "display_name": "GPT-OSS 20B Supervised Fine-Tuning Full Fine-Tuning on GPU", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_override_params_k8s_v1.2.2.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_payload_template_k8s_v1.2.2.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_sft_fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_sft_fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_override_params_sm_jobs_v1.2.2.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_payload_template_sm_jobs_v1.2.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.2.2" + ] + }, + { + "customization_technique": "DPO", + "display_name": "GPT-OSS 20B Direct Preference Optimization Full Fine-Tuning on GPU", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_override_params_k8s_v1.3.2.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_payload_template_k8s_v1.3.2.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_dpo_fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_override_params_sm_jobs_v1.3.2.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_payload_template_sm_jobs_v1.3.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.3.2" + ] + }, + { + "customization_technique": "RLAIF", + "display_name": "GPT OSS 20B GRPO RLAIF Fine-Tuning", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_override_params_k8s_v1.2.3.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_payload_template_k8s_v1.2.3.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "verl-grpo-rlaif-gpt-oss-20b-fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlaif-gpt-oss-20b-fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_override_params_sm_jobs_v1.2.3.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_payload_template_sm_jobs_v1.2.3.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.2.3" + ] + }, + { + "customization_technique": "RLVR", + "display_name": "GPT OSS 20B GRPO RLVR Fine-Tuning", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_override_params_k8s_v1.3.2.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_payload_template_k8s_v1.3.2.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "verl-grpo-rlvr-gpt-oss-20b-fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlvr-gpt-oss-20b-fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "1K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_override_params_sm_jobs_v1.3.2.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_payload_template_sm_jobs_v1.3.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.3.2" + ] + } + ], + "resource_name_base": "openai-reasoning-gpt-oss-20b", + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ], + "training_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/training/v2.0.0/", + "training_dependencies": [], + "training_supported": false, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://huggingface.co/openai/gpt-oss-20b", + "validation_supported": true, + "version": "3.38.0" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json new file mode 100644 index 0000000000..16036c5025 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json @@ -0,0 +1,1093 @@ +{ + "default_inference_instance_type": "ml.m5.large", + "default_training_dataset_key": "training-datasets/tf_flowers/", + "default_training_instance_type": "ml.m5.xlarge", + "deploy_kwargs": {}, + "deprecated": false, + "dynamic_container_deployment_supported": false, + "estimator_kwargs": { + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fine_tuning_supported": true, + "fit_kwargs": {}, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-2": { + "cpu_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-2": { + "alias_ecr_uri_3": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-4": { + "alias_ecr_uri_3": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-6": { + "cpu_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-7": { + "cpu_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-west-1": { + "alias_ecr_uri_3": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-2": { + "alias_ecr_uri_3": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-2": { + "alias_ecr_uri_3": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-3": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "il-central-1": { + "alias_ecr_uri_3": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "mx-central-1": { + "cpu_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-iso-east-1": { + "cpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-inference:1.10.0-gpu-py38" + }, + "us-isob-east-1": { + "cpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "dl1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "i3en": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn2": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "hyperparameters": [ + { + "default": "True", + "name": "train_only_top_layer", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": 5, + "max": 1000, + "min": 1, + "name": "epochs", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.001, + "max": 1, + "min": 1e-08, + "name": "learning_rate", + "scope": "algorithm", + "type": "float" + }, + { + "default": 4, + "max": 1024, + "min": 1, + "name": "batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": "Auto", + "name": "reinitialize_top_layer", + "options": [ + "Auto", + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "name": "sagemaker_submit_directory", + "scope": "container", + "type": "text" + }, + { + "default": "transfer_learning.py", + "name": "sagemaker_program", + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "sagemaker_container_log_level", + "scope": "container", + "type": "text" + } + ], + "incremental_training_supported": true, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": false, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "metrics": [ + { + "Name": "pytorch-ic:val-accuracy", + "Regex": "val_accuracy: ([0-9\\.]+)" + } + ], + "min_sdk_version": "2.189.0", + "model_id": "pytorch-ic-mobilenet-v2", + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/x-image", + "supported_accept_types": [ + "application/json;verbose", + "application/json" + ], + "supported_content_types": [ + "application/x-image" + ] + }, + "provider": "pytorch", + "resource_name_base": "pt-ic-mobilenet-v2", + "supported_inference_instance_types": [ + "ml.m5.large", + "ml.m5.xlarge", + "ml.c5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "supported_training_instance_types": [ + "ml.m5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", + "training_dependencies": [ + "sagemaker_jumpstart_prepack_script_utilities==1.0.0" + ], + "training_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "training_enable_network_isolation": true, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-2": { + "cpu_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-2": { + "alias_ecr_uri_3": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-4": { + "alias_ecr_uri_3": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-6": { + "cpu_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-7": { + "cpu_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-west-1": { + "alias_ecr_uri_3": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-2": { + "alias_ecr_uri_3": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-2": { + "alias_ecr_uri_3": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-3": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "il-central-1": { + "alias_ecr_uri_3": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "mx-central-1": { + "cpu_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-iso-east-1": { + "cpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-training:1.10.0-gpu-py38" + }, + "us-isob-east-1": { + "cpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "dl1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "i3en": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn2": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.1.0", + "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", + "training_supported": true, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", + "validation_supported": false, + "version": "3.0.25" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 4fc1a558ba..1ef21e63da 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1,5 +1,6 @@ """Unit tests for ModelBuilderServers class.""" +import json import os import sys import tempfile @@ -1005,6 +1006,311 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init mock_create.assert_called_once() +class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): + """End-to-end tests for additional model data sources, asserting the + CreateModel request shape. + + Each test uses the documented single-call customer API: + + builder = ModelBuilder.from_jumpstart_config( + jumpstart_config=JumpStartConfig( + model_id="...", accept_eula=True, inference_config_name="..."), + compute=Compute(instance_type="..."), ...) + builder.build() + + and asserts on what the SDK sends to the CreateModel API -- the + ``container_defs`` captured from ``sagemaker_session.create_model`` -- not + on builder internals. This is the request boto receives, so the assertions + cover the full downstream impact: primary container image, model data + source, and AdditionalModelDataSources. + + The spec fixtures under tests/unit/servers/data/jumpstart_specs/ are real, + unmodified specs captured from the production JumpStart content bucket + (jumpstart-cache-prod-us-west-2): + + - pytorch-ic-mobilenet-v2: public model, no additional data sources. + - openai-reasoning-gpt-oss-20b: public model whose default (lmi-optimized) + config carries an ungated EAGLE speculative-decoding source -- the model + that surfaced the propagation bug this suite guards. + - meta-textgeneration-llama-3-1-70b: gated model whose lmi-optimized + config carries a GATED draft_model source + (hosting_eula_key=fmhMetadata/eula/llama3_2Eula.txt). + + Patching is centralized in setUp on the same seams the legacy master-v2 + JumpStartModel tests patched: + + 1. The spec-fetch boundary: JumpStartModelsAccessor.get_model_specs routes + by model id into the captured spec files (master-v2's + PROTOTYPICAL_MODEL_SPECS_DICT pattern); _get_manifest serves the + captured manifest headers. + 2. The AWS boundary: a mock session (whose create_model call is the + assertion target), IAM role validation, artifact staging + (_prepare_for_mode), and the post-create Model.get describe call. + + Everything in between runs for real: JumpStart model-id detection, + JumpStartModelSpecs parsing, inference-config resolution, the + get_init_kwargs factory (image URI resolution, content-bucket injection, + PascalCase shaping), the propagation in _build_for_jumpstart, the + accept_eula application in container_def (shared with the primary model), + and _prepare_container_def_base assembling the CreateModel request. + """ + + SPEC_DIR = Path(__file__).parent / "data" / "jumpstart_specs" + + ROLE_ARN = "arn:aws:iam::123456789012:role/SageMakerRole" + + # The content buckets the sources must resolve into (the raw specs publish + # bucket-less key prefixes): public sources land in the public content + # bucket, gated sources in the private one. + PUBLIC_CONTENT_BUCKET = "jumpstart-cache-prod-us-west-2" + PRIVATE_CONTENT_BUCKET = "jumpstart-private-cache-prod-us-west-2" + + @classmethod + def _spec_additional_source(cls, model_id, config_name="lmi-optimized"): + """The raw additional data source exactly as published in the captured + spec fixture: snake_case keys, bucket-less s3_uri key prefix, and + hosting_eula_key present when the source is gated.""" + with open(cls.SPEC_DIR / f"{model_id}.json") as f: + spec = json.load(f) + sources = spec["inference_config_components"][config_name][ + "hosting_additional_data_sources" + ]["speculative_decoding"] + assert len(sources) == 1, f"expected one source in {model_id}/{config_name}" + return sources[0] + + @classmethod + def _expected_create_model_source(cls, model_id, bucket, accept_eula=None): + """The shape the CreateModel request must carry for the fixture's + source. Data values (channel name, compression, data type, key prefix) + come from the spec fixture so it stays the single source of truth; the + transformation under test is spelled out declaratively here instead of + reusing production code (which would make the assertion a tautology): + + - snake_case spec keys become the PascalCase API keys + - the bucket-less key prefix resolves into the given content bucket + - HostingEulaKey never appears (the API rejects it) + - accept_eula, when set, is folded into S3DataSource.ModelAccessConfig + exactly like the primary model's ModelDataSource + """ + raw = cls._spec_additional_source(model_id) + expected = { + "ChannelName": raw["channel_name"], + "S3DataSource": { + "CompressionType": raw["s3_data_source"]["compression_type"], + "S3DataType": raw["s3_data_source"]["s3_data_type"], + "S3Uri": f"s3://{bucket}/{raw['s3_data_source']['s3_uri']}", + }, + } + if accept_eula is not None: + expected["S3DataSource"]["ModelAccessConfig"] = {"AcceptEula": accept_eula} + return expected + + def setUp(self): + import sagemaker.serve.model_builder as model_builder_module + from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor + from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs + from sagemaker.serve.model_builder import ModelBuilder + + with open(self.SPEC_DIR / "manifest.json") as f: + manifest = [JumpStartModelHeader(header) for header in json.load(f)] + + spec_dir = self.SPEC_DIR + + def get_captured_model_specs(*args, **kwargs): + """Routes spec lookups by model id into the captured real spec + files, like master-v2's PROTOTYPICAL_MODEL_SPECS_DICT loaders.""" + model_id = kwargs.get("model_id") or args[1] + with open(spec_dir / f"{model_id}.json") as f: + return JumpStartModelSpecs(json.load(f)) + + patchers = [ + patch.object( + JumpStartModelsAccessor, "get_model_specs", side_effect=get_captured_model_specs + ), + patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=manifest), + patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), + patch.object( + model_builder_module, "resolve_and_validate_role", return_value=self.ROLE_ARN + ), + patch.object(model_builder_module.Model, "get", return_value=Mock()), + ] + for patcher in patchers: + patcher.start() + self.addCleanup(patcher.stop) + + session = Mock() + session.boto_region_name = "us-west-2" + session.get_caller_identity_arn = Mock(return_value=self.ROLE_ARN) + session.sagemaker_config = {} + session.config = None + session.settings.include_jumpstart_tags = False + session.default_bucket = Mock(return_value="sagemaker-us-west-2-123456789012") + session.default_bucket_prefix = None + session.local_mode = False + self.session = session + + def _build(self, model_id, accept_eula=None, inference_config_name=None, instance_type=None): + """The customer flow, verbatim: one from_jumpstart_config call carrying + the model id, EULA decision, and config selection, then build().""" + from sagemaker.core.jumpstart.configs import JumpStartConfig + from sagemaker.core.training.configs import Compute + from sagemaker.serve.model_builder import ModelBuilder + + builder = ModelBuilder.from_jumpstart_config( + jumpstart_config=JumpStartConfig( + model_id=model_id, + accept_eula=accept_eula, + inference_config_name=inference_config_name, + ), + role_arn=self.ROLE_ARN, + compute=Compute(instance_type=instance_type) if instance_type else None, + sagemaker_session=self.session, + ) + builder.build() + + def _create_model_container_def(self): + """The container definition sent to the CreateModel API.""" + self.session.create_model.assert_called_once() + return self.session.create_model.call_args.kwargs["container_defs"] + + def test_model_without_additional_sources_sends_none_to_create_model(self): + """Base case: the CreateModel request carries no + AdditionalModelDataSources field at all.""" + self._build("pytorch-ic-mobilenet-v2") + + container_def = self._create_model_container_def() + self.assertNotIn("AdditionalModelDataSources", container_def) + + def test_ungated_additional_source_reaches_create_model_without_eula(self): + """The bug this suite guards: an ungated speculative-decoding source + in the model's default config must reach CreateModel even though the + customer never touches accept_eula.""" + self._build("openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge") + + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "openai-reasoning-gpt-oss-20b", bucket=self.PUBLIC_CONTENT_BUCKET + ) + ], + ) + + def test_accept_eula_applies_to_every_model_data_source_uniformly(self): + """The single accept_eula knob folds the same ModelAccessConfig into + the primary model AND each additional source, exactly like the primary + model's ModelDataSource handling. Gatedness is not the SDK's call: the + service determines it from the artifact's bucket and ignores the config + on ungated sources.""" + self._build( + "openai-reasoning-gpt-oss-20b", accept_eula=True, instance_type="ml.g7e.2xlarge" + ) + + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "openai-reasoning-gpt-oss-20b", + bucket=self.PUBLIC_CONTENT_BUCKET, + accept_eula=True, + ) + ], + ) + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": True}, + ) + + def test_gated_additional_source_with_accepted_eula_sends_model_access_config(self): + """Accepting the EULA folds ModelAccessConfig into the gated + additional source AND the primary model in the CreateModel request.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=True, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) + + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "meta-textgeneration-llama-3-1-70b", + bucket=self.PRIVATE_CONTENT_BUCKET, + accept_eula=True, + ) + ], + ) + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": True}, + ) + + def test_gated_additional_source_without_eula_sends_no_model_access_config(self): + """No EULA decision: the request carries the gated source without any + ModelAccessConfig. Enforcement is the service's: the SageMaker control + plane resolves the private-bucket URI as gated and rejects CreateModel + with an EULA validation error, the same as for a gated primary model.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) + + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "meta-textgeneration-llama-3-1-70b", bucket=self.PRIVATE_CONTENT_BUCKET + ) + ], + ) + + def test_gated_additional_source_with_rejected_eula_sends_acceptance_false(self): + """Explicit accept_eula=False is transmitted faithfully as + ModelAccessConfig={"AcceptEula": False} on every source, exactly like + the primary model. The service rejects the gated sources.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=False, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) + + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], + [ + self._expected_create_model_source( + "meta-textgeneration-llama-3-1-70b", + bucket=self.PRIVATE_CONTENT_BUCKET, + accept_eula=False, + ) + ], + ) + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": False}, + ) + + def test_unselected_config_sources_do_not_leak_into_create_model(self): + """Config resolution gates which sources apply: the same gated model on + its default (lmi) config has no additional sources, so the CreateModel + request must not carry the lmi-optimized config's gated draft model.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=True, + instance_type="ml.p4d.24xlarge", + ) + + container_def = self._create_model_container_def() + self.assertNotIn("AdditionalModelDataSources", container_def) + + class TestDeployWrappers(unittest.TestCase): """Test deploy wrapper methods."""