diff --git a/packages/gapic-generator/gapic/generator/generator.py b/packages/gapic-generator/gapic/generator/generator.py index 9fe56aa9de1d..c1932abbe426 100644 --- a/packages/gapic-generator/gapic/generator/generator.py +++ b/packages/gapic-generator/gapic/generator/generator.py @@ -120,7 +120,7 @@ def get_response(self, api_schema: api.API, opts: Options) -> CodeGeneratorRespo for template_name in client_templates: # Quick check: Skip "private" templates. filename = template_name.split("/")[-1] - if filename.startswith("_") and filename != "__init__.py.j2": + if filename.startswith("_") and filename not in ("__init__.py.j2", "_compat.py.j2"): continue # Append to the output files dictionary. diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 new file mode 100644 index 000000000000..5dac8c33a7f9 --- /dev/null +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2 @@ -0,0 +1,271 @@ +# {% include '_license.j2' %} + +"""A compatibility module for older versions of google-api-core.""" + +import functools +import json +import operator +import os +import re +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from google.auth.exceptions import MutualTLSChannelError +import google.protobuf.message + + +try: + from google.api_core.universe import ( + get_default_mtls_endpoint, + get_api_endpoint, + get_universe_domain, + ) +except ImportError: + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Universe domain support was introduced in google-api-core 2.18.0. + # Remove these fallback definitions when google-api-core >= 2.18.0 becomes the minimum required version in generated client setup dependencies. + def get_default_mtls_endpoint(api_endpoint: Optional[str]) -> Optional[str]: + """Converts api endpoint to mTLS endpoint.""" + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + if m is None: + # Could not parse api_endpoint; return as-is. + return api_endpoint + + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + def get_api_endpoint( + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + default_universe: str, + default_mtls_endpoint: Optional[str], + default_endpoint_template: str, + ) -> Optional[str]: + """Return the API endpoint used by the client.""" + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + if universe_domain != default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {default_universe}." + ) + api_endpoint = default_mtls_endpoint + else: + api_endpoint = default_endpoint_template.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint + + def get_universe_domain( + client_universe_domain: Optional[str] = None, + universe_domain_env: Optional[str] = None, + default_universe: str = "googleapis.com", + ) -> str: + """Return the universe domain used by the client.""" + universe_domain = default_universe + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + +try: + from google.api_core.gapic_v1.config import ( + use_client_cert_effective, + get_client_cert_source, + read_environment_variables, + ) +except ImportError: + from google.auth.transport import mtls # type: ignore + + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + + def use_client_cert_effective() -> bool: + """Returns whether client certificate should be used for mTLS.""" + if hasattr(mtls, "should_use_client_cert"): + return mtls.should_use_client_cert() + else: + use_client_cert_str = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + if use_client_cert_str not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be" + " either `true` or `false`" + ) + return use_client_cert_str == "true" + + def get_client_cert_source( + provided_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + use_cert_flag: bool, + ) -> Optional[Callable[[], Tuple[bytes, bytes]]]: + """Return the client cert source to be used by the client.""" + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif ( + hasattr(mtls, "has_default_client_cert_source") + and mtls.has_default_client_cert_source() + ): + client_cert_source = mtls.default_client_cert_source() + else: + raise ValueError( + "Client certificate is required for mTLS, but no client certificate source was provided or found." + ) + return client_cert_source + + def read_environment_variables() -> Tuple[bool, str, Optional[str]]: + """Returns the environment variables used by the client.""" + use_client_cert = use_client_cert_effective() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` " + "must be `never`, `auto` or `always`" + ) + return use_client_cert, use_mtls_endpoint, universe_domain_env + + +try: + from google.api_core.gapic_v1.request import setup_request_id # type: ignore +except ImportError: + # TODO(https://github.com/googleapis/google-cloud-python/issues/17813): Remove this fallback when google-api-core >= 2.26.0 is the minimum required version. + def setup_request_id(request, field_name: str, is_proto3_optional: bool): + """Populate a UUID4 field in the request if it is not already set. + + Args: + request (Union[google.protobuf.message.Message, dict]): The request object. + field_name (str): The name of the field to populate. + is_proto3_optional (bool): Whether the field is proto3 optional. + """ + request_id_val = str(uuid.uuid4()) + if request is None: + return + + if isinstance(request, dict): + if is_proto3_optional: + if field_name not in request or request[field_name] is None: + request[field_name] = request_id_val + elif not request.get(field_name): + request[field_name] = request_id_val + return + + if is_proto3_optional: + try: + # Pure protobuf messages + if not request.HasField(field_name): + setattr(request, field_name, request_id_val) + except (AttributeError, ValueError): + # Proto-plus messages or other objects + if getattr(request, field_name, None) is None: + setattr(request, field_name, request_id_val) + else: + if not getattr(request, field_name, None): + setattr(request, field_name, request_id_val) + + +try: + from google.api_core.rest_helpers import ( + flatten_query_params, + transcode_request, + ) +except ImportError: # pragma: NO COVER + # TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version. + from google.protobuf import json_format # type: ignore + from google.api_core import path_template # type: ignore + + def flatten_query_params(obj, strict=False): # pragma: NO COVER + if obj is not None and not isinstance(obj, dict): + raise TypeError("flatten_query_params must be called with dict object") + return _flatten(obj, key_path=[], strict=strict) + + def _flatten(obj, key_path, strict=False): # pragma: NO COVER + if obj is None: + return [] + if isinstance(obj, dict): + return _flatten_dict(obj, key_path=key_path, strict=strict) + if isinstance(obj, list): + return _flatten_list(obj, key_path=key_path, strict=strict) + return _flatten_value(obj, key_path=key_path, strict=strict) + + def _is_primitive_value(obj): # pragma: NO COVER + if obj is None: + return False + if isinstance(obj, (list, dict)): + raise ValueError("query params may not contain repeated dicts or lists") + return True + + def _flatten_value(obj, key_path, strict=False): # pragma: NO COVER + return [(".".join(key_path), _canonicalize(obj, strict=strict))] + + def _flatten_dict(obj, key_path, strict=False): # pragma: NO COVER + items = ( + _flatten(value, key_path=key_path + [key], strict=strict) + for key, value in obj.items() + ) + return functools.reduce(operator.concat, items, []) + + def _flatten_list(elems, key_path, strict=False): # pragma: NO COVER + items = ( + _flatten_value(elem, key_path=key_path, strict=strict) + for elem in elems + if _is_primitive_value(elem) + ) + return functools.reduce(operator.concat, items, []) + + def _canonicalize(obj, strict=False): # pragma: NO COVER + if strict: + value = str(obj) + if isinstance(obj, bool): + value = value.lower() + return value + return obj + + def transcode_request( # pragma: NO COVER + http_options: List[Dict[str, str]], + request: Any, + required_fields_default_values: Optional[Dict[str, Any]] = None, + rest_numeric_enums: bool = False, + ) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]: + pb_request = getattr(request, "_pb", request) + transcoded_request = path_template.transcode(http_options, pb_request) + + body_json = None + if transcoded_request.get("body") is not None: + body_json = json_format.MessageToJson( + transcoded_request["body"], + use_integers_for_enums=rest_numeric_enums, + ) + + query_params_json = {} + if transcoded_request.get("query_params") is not None: + query_params_json = json.loads(json_format.MessageToJson( + transcoded_request["query_params"], + use_integers_for_enums=rest_numeric_enums, + )) + + if required_fields_default_values: + for k, v in required_fields_default_values.items(): + if k not in query_params_json: + query_params_json[k] = v + + if rest_numeric_enums: + query_params_json["$alt"] = "json;enum-encoding=int" + + return transcoded_request, body_json, query_params_json diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 index e2e3edb24967..8a73595b7789 100644 --- a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 +++ b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 @@ -30,6 +30,7 @@ from google.api_core import exceptions as core_exceptions from google.api_core import extended_operation {% endif %} from google.api_core import gapic_v1 +from {{package_path}} import _compat as client_utils from google.api_core import retry as retries from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport import mtls # type: ignore @@ -143,44 +144,10 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): """{{ service.meta.doc|rst(width=72, indent=4) }}{% if service.version|length %} This class implements API version {{ service.version }}.{% endif %}""" - @staticmethod - def _get_default_mtls_endpoint(api_endpoint) -> Optional[str]: - """Converts api endpoint to mTLS endpoint. - - Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to - "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. - Args: - api_endpoint (Optional[str]): the api endpoint to convert. - Returns: - Optional[str]: converted mTLS api endpoint. - """ - if not api_endpoint: - return api_endpoint - - mtls_endpoint_re = re.compile( - r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" - ) - - m = mtls_endpoint_re.match(api_endpoint) - if m is None: - # Could not parse api_endpoint; return as-is. - return api_endpoint - - name, mtls, sandbox, googledomain = m.groups() - if mtls or not googledomain: - return api_endpoint - - if sandbox: - return api_endpoint.replace( - "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" - ) - - return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") - # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. DEFAULT_ENDPOINT = {% if service.host %}"{{ service.host }}"{% else %}None{% endif %} - DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_MTLS_ENDPOINT = client_utils.get_default_mtls_endpoint( DEFAULT_ENDPOINT ) @@ -392,30 +359,34 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): return client_cert_source @staticmethod - def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint) -> str: + def _get_api_endpoint( + api_override: Optional[str], + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]], + universe_domain: str, + use_mtls_endpoint: str, + ) -> Optional[str]: """Return the API endpoint used by the client. Args: - api_override (str): The API endpoint override. If specified, this is always + api_override (Optional[str]): The API endpoint override. If specified, this is always the return value of this function and the other arguments are not used. - client_cert_source (bytes): The client certificate source used by the client. + client_cert_source (Union[Callable[[], Tuple[bytes, bytes]], None]): The client certificate source used by the client. universe_domain (str): The universe domain used by the client. use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. Possible values are "always", "auto", or "never". Returns: - str: The API endpoint to be used by the client. + Optional[str]: The API endpoint to be used by the client. """ - if api_override is not None: - api_endpoint = api_override - elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): - _default_universe = {{ service.client_name }}._DEFAULT_UNIVERSE - if universe_domain != _default_universe: - raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.") - api_endpoint = {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT - else: - api_endpoint = {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain) - return api_endpoint + return client_utils.get_api_endpoint( + api_override, + client_cert_source, + universe_domain, + use_mtls_endpoint, + {{ service.client_name }}._DEFAULT_UNIVERSE, + {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT, + {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE, + ) @staticmethod def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str: @@ -431,14 +402,11 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): Raises: ValueError: If the universe domain is an empty string. """ - universe_domain = {{ service.client_name }}._DEFAULT_UNIVERSE - if client_universe_domain is not None: - universe_domain = client_universe_domain - elif universe_domain_env is not None: - universe_domain = universe_domain_env - if len(universe_domain.strip()) == 0: - raise ValueError("Universe Domain cannot be an empty string.") - return universe_domain + return client_utils.get_universe_domain( + client_universe_domain, + universe_domain_env, + default_universe={{ service.client_name }}._DEFAULT_UNIVERSE, + ) def _validate_universe_domain(self): """Validates client's and credentials' universe domains are consistent. diff --git a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index c68b390b7c84..7843ceb31bff 100644 --- a/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/packages/gapic-generator/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -156,22 +156,6 @@ def set_event_loop(): asyncio.set_event_loop(None) -def test__get_default_mtls_endpoint(): - api_endpoint = "example.googleapis.com" - api_mtls_endpoint = "example.mtls.googleapis.com" - sandbox_endpoint = "example.sandbox.googleapis.com" - sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" - non_googleapi = "api.example.com" - custom_endpoint = ".custom" - - assert {{ service.client_name }}._get_default_mtls_endpoint(None) is None - assert {{ service.client_name }}._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint - assert {{ service.client_name }}._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint - assert {{ service.client_name }}._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint - assert {{ service.client_name }}._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint - assert {{ service.client_name }}._get_default_mtls_endpoint(non_googleapi) == non_googleapi - assert {{ service.client_name }}._get_default_mtls_endpoint(custom_endpoint) == custom_endpoint - def test__read_environment_variables(): assert {{ service.client_name }}._read_environment_variables() == (False, "auto", None) @@ -313,29 +297,7 @@ def test__get_client_cert_source(): assert {{ service.client_name }}._get_client_cert_source(None, True) is mock_default_cert_source assert {{ service.client_name }}._get_client_cert_source(mock_provided_cert_source, "true") is mock_provided_cert_source -@mock.patch.object({{ service.client_name }}, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template({{ service.client_name }})) -{% if 'grpc' in opts.transport %} -@mock.patch.object({{ service.async_client_name }}, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template({{ service.async_client_name }})) -{% endif %} -def test__get_api_endpoint(): - api_override = "foo.com" - mock_client_cert_source = mock.Mock() - default_universe = {{ service.client_name }}._DEFAULT_UNIVERSE - default_endpoint = {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) - mock_universe = "bar.com" - mock_endpoint = {{ service.client_name }}._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) - - assert {{ service.client_name }}._get_api_endpoint(api_override, mock_client_cert_source, default_universe, "always") == api_override - assert {{ service.client_name }}._get_api_endpoint(None, mock_client_cert_source, default_universe, "auto") == {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT - assert {{ service.client_name }}._get_api_endpoint(None, None, default_universe, "auto") == default_endpoint - assert {{ service.client_name }}._get_api_endpoint(None, None, default_universe, "always") == {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT - assert {{ service.client_name }}._get_api_endpoint(None, mock_client_cert_source, default_universe, "always") == {{ service.client_name }}.DEFAULT_MTLS_ENDPOINT - assert {{ service.client_name }}._get_api_endpoint(None, None, mock_universe, "never") == mock_endpoint - assert {{ service.client_name }}._get_api_endpoint(None, None, default_universe, "never") == default_endpoint - with pytest.raises(MutualTLSChannelError) as excinfo: - {{ service.client_name }}._get_api_endpoint(None, mock_client_cert_source, mock_universe, "auto") - assert str(excinfo.value) == "mTLS is not supported in any universe other than googleapis.com." {% if service.version %} {% for method in service.methods.values() %}{% with method_name = method.name|snake_case %} @@ -384,17 +346,7 @@ def test_{{ method_name }}_api_version_header(transport_name): {% endfor %} {% endif %}{# service.version #} -def test__get_universe_domain(): - client_universe_domain = "foo.com" - universe_domain_env = "bar.com" - - assert {{ service.client_name }}._get_universe_domain(client_universe_domain, universe_domain_env) == client_universe_domain - assert {{ service.client_name }}._get_universe_domain(None, universe_domain_env) == universe_domain_env - assert {{ service.client_name }}._get_universe_domain(None, None) == {{ service.client_name }}._DEFAULT_UNIVERSE - with pytest.raises(ValueError) as excinfo: - {{ service.client_name }}._get_universe_domain("", None) - assert str(excinfo.value) == "Universe Domain cannot be an empty string." @pytest.mark.parametrize("error_code,cred_info_json,show_cred_info", [ (401, CRED_INFO_JSON, True), diff --git a/packages/gapic-generator/noxfile.py b/packages/gapic-generator/noxfile.py index 52209f41ff38..dce1cf1504e2 100644 --- a/packages/gapic-generator/noxfile.py +++ b/packages/gapic-generator/noxfile.py @@ -181,6 +181,7 @@ def fragment(session, use_ads_templates=False): "grpcio-tools", ) session.install("-e", ".") + session.install("-e", "../google-api-core") # The specific failure is `Plugin output is unparseable` if session.python == "3.10": @@ -245,6 +246,7 @@ def showcase_library( # Install gapic-generator-python session.install("-e", ".") + session.install("-e", "../google-api-core") # Install grpcio-tools for protoc session.install("grpcio-tools") diff --git a/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_asset_service.py b/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_asset_service.py index ea110a38acc3..1a367689d14e 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_asset_service.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/tests/unit/gapic/asset_v1/test_asset_service.py @@ -700,7 +700,7 @@ def test_asset_service_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -747,7 +747,7 @@ def test_asset_service_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_iam_credentials.py b/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_iam_credentials.py index 14a6074a40f9..508e17a7e889 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_iam_credentials.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/tests/unit/gapic/credentials_v1/test_iam_credentials.py @@ -690,7 +690,7 @@ def test_iam_credentials_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -737,7 +737,7 @@ def test_iam_credentials_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_eventarc.py b/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_eventarc.py index 533e401eb1e7..b9495ef6e42d 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_eventarc.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/tests/unit/gapic/eventarc_v1/test_eventarc.py @@ -721,7 +721,7 @@ def test_eventarc_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -768,7 +768,7 @@ def test_eventarc_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_config_service_v2.py b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_config_service_v2.py index eada5b433c55..8cc17d810664 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_config_service_v2.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_config_service_v2.py @@ -678,7 +678,7 @@ def test_config_service_v2_client_get_mtls_endpoint_and_cert_source(client_class config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -725,7 +725,7 @@ def test_config_service_v2_client_get_mtls_endpoint_and_cert_source(client_class config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_logging_service_v2.py b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_logging_service_v2.py index 65559a5d1073..fd56f0210d78 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_logging_service_v2.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_logging_service_v2.py @@ -679,7 +679,7 @@ def test_logging_service_v2_client_get_mtls_endpoint_and_cert_source(client_clas config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -726,7 +726,7 @@ def test_logging_service_v2_client_get_mtls_endpoint_and_cert_source(client_clas config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_metrics_service_v2.py b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_metrics_service_v2.py index 90cdab2be2b2..762b4b3ab94d 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_metrics_service_v2.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/tests/unit/gapic/logging_v2/test_metrics_service_v2.py @@ -677,7 +677,7 @@ def test_metrics_service_v2_client_get_mtls_endpoint_and_cert_source(client_clas config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -724,7 +724,7 @@ def test_metrics_service_v2_client_get_mtls_endpoint_and_cert_source(client_clas config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_config_service_v2.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_config_service_v2.py index 9eec837e6f58..93731051d1bc 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_config_service_v2.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_config_service_v2.py @@ -678,7 +678,7 @@ def test_base_config_service_v2_client_get_mtls_endpoint_and_cert_source(client_ config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -725,7 +725,7 @@ def test_base_config_service_v2_client_get_mtls_endpoint_and_cert_source(client_ config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_logging_service_v2.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_logging_service_v2.py index 65559a5d1073..fd56f0210d78 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_logging_service_v2.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_logging_service_v2.py @@ -679,7 +679,7 @@ def test_logging_service_v2_client_get_mtls_endpoint_and_cert_source(client_clas config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -726,7 +726,7 @@ def test_logging_service_v2_client_get_mtls_endpoint_and_cert_source(client_clas config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_metrics_service_v2.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_metrics_service_v2.py index 310677b64bc6..090ea4a91bec 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_metrics_service_v2.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/tests/unit/gapic/logging_v2/test_metrics_service_v2.py @@ -677,7 +677,7 @@ def test_base_metrics_service_v2_client_get_mtls_endpoint_and_cert_source(client config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -724,7 +724,7 @@ def test_base_metrics_service_v2_client_get_mtls_endpoint_and_cert_source(client config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_cloud_redis.py b/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_cloud_redis.py index 8ca1fb5194a6..d58d99d1fdb8 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_cloud_redis.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/tests/unit/gapic/redis_v1/test_cloud_redis.py @@ -708,7 +708,7 @@ def test_cloud_redis_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -755,7 +755,7 @@ def test_cloud_redis_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_cloud_redis.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_cloud_redis.py index 3f6b7aa521f3..780964608350 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_cloud_redis.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/tests/unit/gapic/redis_v1/test_cloud_redis.py @@ -708,7 +708,7 @@ def test_cloud_redis_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -755,7 +755,7 @@ def test_cloud_redis_client_get_mtls_endpoint_and_cert_source(client_class): config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test__compat.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test__compat.py new file mode 100644 index 000000000000..e10021c90673 --- /dev/null +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test__compat.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import builtins +import sys + +import mock +import pytest + +from google.cloud.storagebatchoperations_v1 import _compat + + +def test_setup_request_id_fallback(): + with mock.patch("sys.modules", dict(sys.modules)): + if "google.api_core.gapic_v1.request" in sys.modules: + del sys.modules["google.api_core.gapic_v1.request"] + + real_import = builtins.__import__ + + def mock_import(name, globals=None, locals=None, fromlist=(), level=0): + if name == "google.api_core.gapic_v1.request": + raise ImportError("Mocked ImportError for gapic_v1.request") + return real_import(name, globals, locals, fromlist, level) + + with mock.patch("builtins.__import__", side_effect=mock_import): + # Reload _compat to trigger the fallback path + import importlib + importlib.reload(_compat) + + # The fallback method should be defined + assert hasattr(_compat, "setup_request_id") + + # Test it works on a dict + request = {} + _compat.setup_request_id(request, "request_id", True) + assert "request_id" in request + + # Test it works on a dict when proto3 optional is False + request = {} + _compat.setup_request_id(request, "request_id", False) + assert "request_id" in request + + # Reset the module state + importlib.reload(_compat) diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py index 5c53e97f8d12..892375775385 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/tests/unit/gapic/storagebatchoperations_v1/test_storage_batch_operations.py @@ -776,7 +776,7 @@ def test_storage_batch_operations_client_get_mtls_endpoint_and_cert_source(clien config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): @@ -823,7 +823,7 @@ def test_storage_batch_operations_client_get_mtls_endpoint_and_cert_source(clien config_filename = "mock_certificate_config.json" config_file_content = json.dumps(config_data) m = mock.mock_open(read_data=config_file_content) - with mock.patch("builtins.open", m): + with mock.patch("builtins.open", m), mock.patch("os.path.exists", side_effect=lambda path: os.path.basename(path) == config_filename): with mock.patch.dict( os.environ, {"GOOGLE_API_CERTIFICATE_CONFIG": config_filename} ): diff --git a/packages/gapic-generator/tests/unit/generator/test_generator.py b/packages/gapic-generator/tests/unit/generator/test_generator.py index 9d8545c4192f..f1566ee6e5a7 100644 --- a/packages/gapic-generator/tests/unit/generator/test_generator.py +++ b/packages/gapic-generator/tests/unit/generator/test_generator.py @@ -117,6 +117,8 @@ def test_get_response_ignores_private_files(): list_templates.return_value = [ "foo/bar/baz.py.j2", "foo/bar/_base.py.j2", + "foo/bar/__init__.py.j2", + "foo/bar/_compat.py.j2", "molluscs/squid/sample.py.j2", ] with mock.patch.object(jinja2.Environment, "get_template") as get_template: @@ -128,12 +130,13 @@ def test_get_response_ignores_private_files(): get_template.assert_has_calls( [ mock.call("molluscs/squid/sample.py.j2"), + mock.call("foo/bar/__init__.py.j2"), + mock.call("foo/bar/_compat.py.j2"), mock.call("foo/bar/baz.py.j2"), - ] + ], + any_order=True, ) - assert len(cgr.file) == 1 - assert cgr.file[0].name == "foo/bar/baz.py" - assert cgr.file[0].content == "I am a template result.\n" + assert len(cgr.file) == 3 def test_get_response_fails_invalid_file_paths(): diff --git a/packages/google-api-core/google/api_core/gapic_v1/requests.py b/packages/google-api-core/google/api_core/gapic_v1/requests.py index 76f5e916716d..82a2fbac39ec 100644 --- a/packages/google-api-core/google/api_core/gapic_v1/requests.py +++ b/packages/google-api-core/google/api_core/gapic_v1/requests.py @@ -65,6 +65,13 @@ def setup_request_id( setattr(request, field_name, str(uuid.uuid4())) except (AttributeError, ValueError): # Proto-plus messages or other objects + if hasattr(request, "_pb"): + try: + if not request._pb.HasField(field_name): + setattr(request, field_name, str(uuid.uuid4())) + return + except (AttributeError, ValueError): + pass if getattr(request, field_name, None) is None: setattr(request, field_name, str(uuid.uuid4())) else: diff --git a/packages/google-api-core/tests/unit/gapic/test_requests.py b/packages/google-api-core/tests/unit/gapic/test_requests.py index a69bff5e16d7..f03843b1d6d7 100644 --- a/packages/google-api-core/tests/unit/gapic/test_requests.py +++ b/packages/google-api-core/tests/unit/gapic/test_requests.py @@ -113,15 +113,18 @@ def test_setup_request_id(request_obj, is_proto3_optional, expected): assert value == expected -def test_setup_request_id_assertion_strictness(mocker): - # Mock uuid.uuid4 to return a UUID with trailing characters - mock_uuid = mocker.patch("uuid.uuid4") - mock_uuid.return_value.__str__.return_value = ( - "12345678-1234-4123-8123-123456789012-extra" - ) +from unittest import mock - # We expect test_setup_request_id to fail (raise AssertionError) because the UUID is invalid. - # If test_setup_request_id uses re.fullmatch, it will fail (raise AssertionError), so pytest.raises passes. - # If test_setup_request_id uses re.match, it will NOT fail, so pytest.raises fails! - with pytest.raises(AssertionError): - test_setup_request_id(MockRequest(), is_proto3_optional=True, expected="uuid") + +def test_setup_request_id_assertion_strictness(): + # Mock uuid.uuid4 to return a UUID with trailing characters + with mock.patch("uuid.uuid4") as mock_uuid: + mock_uuid.return_value.__str__.return_value = ( + "12345678-1234-4123-8123-123456789012-extra" + ) + + # We expect test_setup_request_id to fail (raise AssertionError) because the UUID is invalid. + # If test_setup_request_id uses re.fullmatch, it will fail (raise AssertionError), so pytest.raises passes. + # If test_setup_request_id uses re.match, it will NOT fail, so pytest.raises fails! + with pytest.raises(AssertionError): + test_setup_request_id(MockRequest(), is_proto3_optional=True, expected="uuid")