gRPC Service Improvements (#106)

Author: cretz

.github/workflows/ci.yml

@@ -51,7 +51,7 @@ jobs:
         run: |
           poe gen-protos
           poe format
-          [[ -z $(git status --porcelain) ]] || (git status; echo "Protos changed"; exit 1)
+          [[ -z $(git status --porcelain) ]] || (git diff; echo "Protos changed"; exit 1)
 
       # Do docs stuff (only on one host)
       - name: Build API docs

README.md

@@ -51,11 +51,6 @@ These steps can be followed to use with a virtual environment and `pip`:
   * Needed because older versions of `pip` may not pick the right wheel
 * Install Temporal SDK - `python -m pip install temporalio`
 
-For macOS M1 users on Python 3.10+, due to an
-[improperly named wheel in PyPI for gRPC](https://github.com/grpc/grpc/issues/28387), you may have to run
-`pip install --no-binary :all: grpcio --ignore-installed` in your environment before use. (this is fixed in the next
-version of `temporalio` and in the `main` branch)
-
 The SDK is now ready for use. To build from source, see "Building" near the end of this documentation.
 
 **NOTE: This README is for the current branch and not necessarily what's released on `PyPI`.**

poetry.lock

@@ -25,7 +25,7 @@ script = "build.py"
 "Bug Tracker" = "https://github.com/temporalio/sdk-python/issues"
 
 [tool.poetry.dependencies]
-grpcio = "^1.47.0"
+grpcio = { version = "^1.48.0", optional = true }
 opentelemetry-api = { version = "^1.11.1", optional = true }
 opentelemetry-sdk = { version = "^1.11.1", optional = true }
 protobuf = "^3.20.1"
@@ -36,10 +36,11 @@ typing-extensions = "^4.2.0"
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"
 cibuildwheel = "^2.7.0"
-grpcio-tools = "^1.47.0"
+grpcio-tools = "^1.48.0"
 isort = "^5.10.1"
 mypy = "^0.961"
 mypy-protobuf = "^3.2.0"
+protoc-wheel-0 = "^21.1"
 pydantic = "^1.9.1"
 pydocstyle = "^6.1.1"
 # TODO(cretz): Update when https://github.com/twisted/pydoctor/pull/595 released
@@ -48,13 +49,14 @@ pydoctor = { git = "https://github.com/cretz/pydoctor.git", branch = "overloads"
 pytest = "^7.1.2"
 pytest-asyncio = "^0.18.3"
 pytest-timeout = "^2.1.0"
-setuptools = "^62.3.3"
+setuptools = "^64.0.1"
 setuptools-rust = "^1.3.0"
 toml = "^0.10.2"
 twine = "^4.0.1"
 
 [tool.poetry.extras]
 opentelemetry = ["opentelemetry-api", "opentelemetry-sdk"]
+grpc = ["grpc"]
 
 [tool.poe.tasks]
 build-develop = ["build-bridge-develop"]
@@ -94,6 +96,7 @@ asyncio_mode = "auto"
 log_cli = true
 log_cli_level = "INFO"
 log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
+testpaths = ["tests"]
 timeout = 600
 timeout_func_only = true
 
@@ -158,4 +161,4 @@ sidebar-expand-depth = 2
 
 [build-system]
 build-backend = "poetry.core.masonry.api"
-requires = ["poetry-core>=1.0.0", "setuptools", "wheel", "setuptools-rust"]
+requires = ["poetry-core>=1.0.0", "setuptools", "wheel", "setuptools-rust", "protoc-wheel-0"]

pyproject.toml

@@ -12,7 +12,14 @@
 proto_dir = base_dir / "temporalio" / "bridge" / "sdk-core" / "protos"
 api_proto_dir = proto_dir / "api_upstream"
 core_proto_dir = proto_dir / "local"
-proto_paths = proto_dir.glob("**/*.proto")
+testsrv_proto_dir = proto_dir / "testsrv_upstream"
+
+# Exclude testsrv dependencies
+proto_paths = (
+    v
+    for v in proto_dir.glob("**/*.proto")
+    if not str(v).startswith(str(testsrv_proto_dir / "dependencies"))
+)
 
 api_out_dir = base_dir / "temporalio" / "api"
 sdk_out_dir = base_dir / "temporalio" / "bridge" / "proto"
@@ -71,25 +78,39 @@ def fix_generated_output(base_path: Path):
         # Add docstring to API's init
         if str(base_path.as_posix()).endswith("/temporal/api"):
             f.write('"""gRPC API."""\n')
-        # Imports
+        # Non-gRPC imports
         message_names = []
         for stem, messages in imports.items():
-            for message in messages:
-                # MyPy protobuf does not document this experimental class, see
-                # https://github.com/nipunn1313/mypy-protobuf/issues/212#issuecomment-885300106
-                import_suffix = ""
-                if stem == "service_pb2_grpc" and (
-                    message == "WorkflowService" or message == "OperatorService"
-                ):
-                    import_suffix = " # type: ignore"
-                f.write(f"from .{stem} import {message}{import_suffix}\n")
-                message_names.append(message)
+            if stem != "service_pb2_grpc":
+                for message in messages:
+                    f.write(f"from .{stem} import {message}\n")
+                    message_names.append(message)
         # __all__
         message_names = sorted(message_names)
         if message_names:
             f.write(
                 f'\n__all__ = [\n    "' + '",\n    "'.join(message_names) + '",\n]\n'
             )
+        # gRPC imports
+        if "service_pb2_grpc" in imports:
+            message_names = []
+            f.write("\n# gRPC is optional\ntry:\n    import grpc\n")
+            for message in imports["service_pb2_grpc"]:
+                # MyPy protobuf does not document this experimental class, see
+                # https://github.com/nipunn1313/mypy-protobuf/issues/212#issuecomment-885300106
+                import_suffix = ""
+                if (
+                    message == "WorkflowService"
+                    or message == "OperatorService"
+                    or message == "TestService"
+                ):
+                    import_suffix = " # type: ignore"
+                f.write(f"    from .service_pb2_grpc import {message}{import_suffix}\n")
+                message_names.append(message)
+            # __all__
+            message_names = sorted(message_names)
+            f.write(f'    __all__.extend(["' + '", "'.join(message_names) + '"])\n')
+            f.write("except ImportError:\n    pass")
 
 
 if __name__ == "__main__":
@@ -102,6 +123,7 @@ def fix_generated_output(base_path: Path):
                 "-mgrpc_tools.protoc",
                 f"--proto_path={api_proto_dir}",
                 f"--proto_path={core_proto_dir}",
+                f"--proto_path={testsrv_proto_dir}",
                 f"--python_out={temp_dir}",
                 f"--grpc_python_out={temp_dir}",
                 f"--mypy_out={temp_dir}",

scripts/gen_protos.py

@@ -1,5 +1,5 @@
 """Python SDK for Temporal."""
 
-from .workflow_service import __version__ as __sdk_version
+from .service import __version__ as __sdk_version
 
 __version__ = __sdk_version

temporalio/__init__.py

@@ -1,8 +1,10 @@
 from .message_pb2 import (
+    AcceptWorkflowUpdateCommandAttributes,
     CancelTimerCommandAttributes,
     CancelWorkflowExecutionCommandAttributes,
     Command,
     CompleteWorkflowExecutionCommandAttributes,
+    CompleteWorkflowUpdateCommandAttributes,
     ContinueAsNewWorkflowExecutionCommandAttributes,
     FailWorkflowExecutionCommandAttributes,
     RecordMarkerCommandAttributes,
@@ -16,10 +18,12 @@
 )
 
 __all__ = [
+    "AcceptWorkflowUpdateCommandAttributes",
     "CancelTimerCommandAttributes",
     "CancelWorkflowExecutionCommandAttributes",
     "Command",
     "CompleteWorkflowExecutionCommandAttributes",
+    "CompleteWorkflowUpdateCommandAttributes",
     "ContinueAsNewWorkflowExecutionCommandAttributes",
     "FailWorkflowExecutionCommandAttributes",
     "RecordMarkerCommandAttributes",