cleanup: refactor rest transport class in gapics (#2099)

Author: ohmayr

gapic/generator/generator.py

@@ -297,6 +297,8 @@ def _render_template(
                         # TODO(yon-mg) - remove when rest async implementation resolved
                         # temporarily stop async client gen while rest async is unkown
                         ('async' in template_name and 'grpc' not in opts.transport)
+                        or
+                        ('rest_base' in template_name and 'rest' not in opts.transport)
                 ):
                     continue
 
@@ -319,7 +321,7 @@ def _render_template(
 
     def _is_desired_transport(self, template_name: str, opts: Options) -> bool:
         """Returns true if template name contains a desired transport"""
-        desired_transports = ['__init__', 'base'] + opts.transport
+        desired_transports = ['__init__', 'base', 'README'] + opts.transport
         return any(transport in template_name for transport in desired_transports)
 
     def _get_file(

gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2

@@ -68,3 +68,94 @@ except ImportError:  # pragma: NO COVER
             )
     {% endif %}{# service_version #}
 {% endmacro %}
+
+{% macro operations_mixin_imports(api, service, opts) %}
+{% if import_ns is not defined %}
+{% set import_ns = namespace(has_operations_mixin=false) %}
+{% endif %}{# import_ns is not defined #}
+{% set import_ns.has_operations_mixin = api.has_operations_mixin %}
+
+{% filter sort_lines %}
+{% for method in service.methods.values() %}
+{{method.input.ident.python_import}}
+{% if method.output.ident|string() == "operations_pb2.Operation" %}
+{% set import_ns.has_operations_mixin = True %}
+{% else %}
+{{method.output.ident.python_import}}
+{% endif %}
+{% endfor %}
+{% if opts.add_iam_methods %}
+from google.iam.v1 import iam_policy_pb2  # type: ignore
+from google.iam.v1 import policy_pb2  # type: ignore
+{% endif %}{# opts.add_iam_methods #}
+{% endfilter %}
+{% if import_ns.has_operations_mixin %}
+from google.longrunning import operations_pb2  # type: ignore
+{% endif %}{# import_ns.has_operations_mixin #}
+{% endmacro %}
+
+{% macro http_options_method(rules) %}
+@staticmethod
+def _get_http_options():
+    http_options: List[Dict[str, str]] = [
+    {%- for rule in rules %}{
+        'method': '{{ rule.method }}',
+        'uri': '{{ rule.uri }}',
+        {% if rule.body %}
+        'body': '{{ rule.body }}',
+        {% endif %}{# rule.body #}
+    },
+    {% endfor %}{# rule in rules #}
+    ]
+    return http_options
+{% endmacro %}
+
+{% macro response_method(body_spec) %}
+@staticmethod
+def _get_response(
+    host,
+    metadata,
+    query_params,
+    session,
+    timeout,
+    transcoded_request,
+    body=None):
+    
+    uri = transcoded_request['uri']
+    method = transcoded_request['method']
+    headers = dict(metadata)
+    headers['Content-Type'] = 'application/json'
+    response = getattr(session, method)(
+        "{host}{uri}".format(host=host, uri=uri),
+        timeout=timeout,
+        headers=headers,
+        params=rest_helpers.flatten_query_params(query_params, strict=True),
+        {% if body_spec %}
+        data=body,
+        {% endif %}
+        )
+    return response
+{% endmacro %}
+
+{% macro rest_call_method_common(body_spec, method_name, service_name) %}
+
+    http_options = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_http_options()
+    request, metadata = self._interceptor.pre_{{ method_name|snake_case }}(request, metadata)
+    transcoded_request = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_transcoded_request(http_options, request)
+
+    {% if body_spec %}
+    body = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_request_body_json(transcoded_request)
+    {% endif %} {# body_spec #}
+
+    # Jsonify the query params
+    query_params = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_query_params_json(transcoded_request)
+
+    # Send the request
+    response = {{ service_name }}RestTransport._{{method_name}}._get_response(self._host, metadata, query_params, self._session, timeout, transcoded_request{% if body_spec %}, body{% endif %})
+
+    # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
+    # subclass.
+    if response.status_code >= 400:
+        raise core_exceptions.from_http_response(response)
+
+{% endmacro %}

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/README.rst.j2

@@ -0,0 +1,10 @@
+
+transport inheritance structure
+_______________________________
+
+`{{ service.name }}Transport` is the ABC for all transports.
+- public child `{{ service.name }}GrpcTransport` for sync gRPC transport (defined in `grpc.py`).
+- public child `{{ service.name }}GrpcAsyncIOTransport` for async gRPC transport (defined in `grpc_asyncio.py`).
+- private child `_Base{{ service.name }}RestTransport` for base REST transport with inner classes `_BaseMETHOD` (defined in `rest_base.py`).
+- public child `{{ service.name }}RestTransport` for sync REST transport with inner classes `METHOD` derived from the parent's corresponding `_BaseMETHOD` classes (defined in `rest.py`).
+{# Since the service mixins have a similar structure, we factor out shared code into `_shared_macros.j2` to avoid duplication. #}

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_mixins.py.j2

@@ -1,3 +1,21 @@
+{#
+ # Copyright (C) 2024  Google LLC
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ #     http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+#}
+    
+{% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
+
     {% if "grpc" in opts.transport %}
 
     {% if api.has_operations_mixin %}

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins.py.j2

@@ -1,11 +1,32 @@
+{#
+ # Copyright (C) 2024  Google LLC
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ #     http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+#}
+    
+{% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
+
     {% if "rest" in opts.transport %}
 
     {% for name, sig in api.mixin_api_signatures.items() %} 
     @property
     def {{ name|snake_case }}(self):
         return self._{{ name }}(self._session, self._host, self._interceptor) # type: ignore
 
-    class _{{ name }}({{service.name}}RestStub):
+    class _{{ name }}(_Base{{ service.name }}RestTransport._Base{{name}}, {{service.name}}RestStub):
+        {% set body_spec = api.mixin_http_options["{}".format(name)][0].body %}
+        {{ shared_macros.response_method(body_spec)|indent(8) }}
+        
         def __call__(self,
             request: {{ sig.request_type }}, *,
             retry: OptionalRetry=gapic_v1.method.DEFAULT,
@@ -32,52 +53,7 @@
                 {{ sig.response_type }}: Response from {{ name }} method.
             {% endif %}
             """
-
-            http_options: List[Dict[str, str]] = [
-            {%- for rule in api.mixin_http_options["{}".format(name)] %}{
-                'method': '{{ rule.method }}',
-                'uri': '{{ rule.uri }}',
-                {% if rule.body %}
-                'body': '{{ rule.body }}',
-                {% endif %}{# rule.body #}
-            },
-            {% endfor %}
-            ]
-
-            request, metadata = self._interceptor.pre_{{ name|snake_case }}(request, metadata)
-            request_kwargs = json_format.MessageToDict(request)
-            transcoded_request = path_template.transcode(
-                http_options, **request_kwargs)
-
-            {% set body_spec = api.mixin_http_options["{}".format(name)][0].body %}
-            {%- if body_spec %}
-            body = json.dumps(transcoded_request['body'])
-            {%- endif %}
-
-            uri = transcoded_request['uri']
-            method = transcoded_request['method']
-
-            # Jsonify the query params
-            query_params = json.loads(json.dumps(transcoded_request['query_params']))
-
-            # Send the request
-            headers = dict(metadata)
-            headers['Content-Type'] = 'application/json'
-
-            response = getattr(self._session, method)(
-                "{host}{uri}".format(host=self._host, uri=uri),
-                timeout=timeout,
-                headers=headers,
-                params=rest_helpers.flatten_query_params(query_params),
-                {% if body_spec %}
-                data=body,
-                {% endif %}
-            )
-
-            # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
-            # subclass.
-            if response.status_code >= 400:
-                raise core_exceptions.from_http_response(response)
+            {{ shared_macros.rest_call_method_common(body_spec, name, service.name)|indent(8) }}
 
             {% if sig.response_type == "None" %}
             return self._interceptor.post_{{ name|snake_case }}(None)

gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2

@@ -0,0 +1,49 @@
+{#
+ # Copyright (C) 2024  Google LLC
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ #     http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+#}
+
+{% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
+    
+    {% if "rest" in opts.transport %}
+
+    {% for name, sig in api.mixin_api_signatures.items() %}
+    class _Base{{ name }}:
+        
+        {{ shared_macros.http_options_method(api.mixin_http_options["{}".format(name)])|indent(8)}}
+
+        @staticmethod
+        def _get_transcoded_request(http_options, request):
+            request_kwargs = json_format.MessageToDict(request)
+            transcoded_request = path_template.transcode(
+                http_options, **request_kwargs)
+            return transcoded_request
+
+        {% set body_spec = api.mixin_http_options["{}".format(name)][0].body %}
+        {%- if body_spec %}
+
+        @staticmethod
+        def _get_request_body_json(transcoded_request):
+            body = json.dumps(transcoded_request['body'])
+            return body
+
+        {%- endif %} {# body_spec #}
+
+        @staticmethod
+        def _get_query_params_json(transcoded_request):
+            query_params = json.loads(json.dumps(transcoded_request['query_params']))
+            return query_params
+
+    {% endfor %}
+    {% endif %} {# rest in opts.transport #}