feat: allow configuring gRPC keepalive (#478)

Signed-off-by: Thibaud Martinez <thibaud.martinez@owkin.com>
Signed-off-by: Aurélien Gasser <aurelien.gasser@gmail.com>

Author: thibowk

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.31.2] 2022-09-30
+
+### Added
+
+- Allow configuring gRPC keepalive
+
 ## [0.31.1] 2022-09-29
 
 ### Fixed

backend/backend/settings/common.py

@@ -131,7 +131,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str:
     "django_structlog.middlewares.CeleryMiddleware",
 ]
 
-
 DJANGO_LOG_SQL_QUERIES = to_bool(os.environ.get("DJANGO_LOG_SQL_QUERIES", "True"))
 if DJANGO_LOG_SQL_QUERIES:
     MIDDLEWARE.append("libs.sql_printing_middleware.SQLPrintingMiddleware")
@@ -282,7 +281,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str:
 WORKER_PVC_SUBTUPLE = os.environ.get("WORKER_PVC_SUBTUPLE")
 WORKER_REPLICA_SET_NAME = os.environ.get("WORKER_REPLICA_SET_NAME")
 
-
 NAMESPACE = os.getenv("NAMESPACE")
 HOSTNAME = os.getenv("HOSTNAME")
 
@@ -316,7 +314,6 @@ def build_broker_url(user: str, password: str, host: str, port: str) -> str:
         + ["django_prometheus.middleware.PrometheusAfterMiddleware"]
     )
 
-
 DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000
 
 # Uploaded file max size, in bytes

backend/backend/settings/deps/orchestrator.py

@@ -10,4 +10,29 @@
 ORCHESTRATOR_TLS_CLIENT_CERT_PATH = os.environ.get("ORCHESTRATOR_TLS_CLIENT_CERT_PATH")
 ORCHESTRATOR_TLS_CLIENT_KEY_PATH = os.environ.get("ORCHESTRATOR_TLS_CLIENT_KEY_PATH")
 ORCHESTRATOR_RETRY_DELAY = 1
-ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS = int(os.environ.get("ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS", "5000"))
+
+# @ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS: The period (in milliseconds) after which a keepalive ping is sent on the
+# transport.
+#
+# NB: For keepalive to work properly and as intended, the client-side keepalive settings should be in agreement
+# with the server-side settings. If a client sends pings more often than the server is willing to accept,
+# the connection will be terminated with a GOAWAY frame with "too_many_pings" as the debug data.
+ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS = int(os.getenv("ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS", "60000"))
+
+# @ORCHESTRATOR_GRPC_KEEPALIVE_TIMEOUT_MS: The amount of time (in milliseconds) the sender of the keepalive ping
+# waits for an acknowledgement. If it does not receive an acknowledgement within this time, it will close
+# the connection.
+ORCHESTRATOR_GRPC_KEEPALIVE_TIMEOUT_MS = int(os.getenv("ORCHESTRATOR_GRPC_KEEPALIVE_TIMEOUT_MS", "20000"))
+
+# @ORCHESTRATOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS: If set to 1 (0: false; 1: true), allows keepalive pings
+# to be sent even if there are no calls in flight.
+ORCHESTRATOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS = bool(
+    int(os.getenv("ORCHESTRATOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS", "0"))
+)
+
+# @ORCHESTRATOR_GRPC_KEEPALIVE_MAX_PINGS_WITHOUT_DATA The maximum number of pings that can be sent when there is
+# no data/header frame to be sent. gRPC Core will not continue sending pings if we run over the limit.
+# Setting it to 0 allows sending pings without such a restriction.
+ORCHESTRATOR_GRPC_KEEPALIVE_MAX_PINGS_WITHOUT_DATA = int(
+    os.getenv("ORCHESTRATOR_GRPC_KEEPALIVE_MAX_PINGS_WITHOUT_DATA", "0")
+)

backend/backend/settings/test.py

@@ -19,7 +19,10 @@
 ORCHESTRATOR_TLS_ENABLED = False
 ORCHESTRATOR_MTLS_ENABLED = False
 ORCHESTRATOR_RETRY_DELAY = 0
-ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS = 10000
+ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS = 60000
+ORCHESTRATOR_GRPC_KEEPALIVE_TIMEOUT_MS = 20000
+ORCHESTRATOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS = False
+ORCHESTRATOR_GRPC_KEEPALIVE_MAX_PINGS_WITHOUT_DATA = 0
 
 LEDGER_MSP_ID = "testOrgMSP"
 LEDGER_CHANNELS = {"mychannel": {"chaincode": {"name": "mycc"}}}

backend/substrapp/orchestrator.py

@@ -23,6 +23,11 @@ def get_orchestrator_client(channel_name: str = None) -> OrchestratorClient:
     if channel_name is not None:
         chaincode = settings.LEDGER_CHANNELS[channel_name]["chaincode"]["name"]
 
-    opts = (("grpc.keepalive_time_ms", settings.ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS),)
+    opts = (
+        ("grpc.keepalive_time_ms", settings.ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS),
+        ("grpc.keepalive_timeout_ms", settings.ORCHESTRATOR_GRPC_KEEPALIVE_TIMEOUT_MS),
+        ("grpc.keepalive_permit_without_calls", settings.ORCHESTRATOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS),
+        ("grpc.http2.max_pings_without_data", settings.ORCHESTRATOR_GRPC_KEEPALIVE_MAX_PINGS_WITHOUT_DATA),
+    )
 
     return OrchestratorClient(host, channel_name, mspid, chaincode, cacert, client_key, client_cert, opts=opts)

docs/settings.md

@@ -72,7 +72,10 @@ Accepted true values for `bool` are: `1`, `ON`, `On`, `on`, `T`, `t`, `TRUE`, `T
 
 | Type | Setting | Default value | Comment |
 |------|---------|---------------|---------|
-| int | `ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS` | `5000` |  |
+| int | `ORCHESTRATOR_GRPC_KEEPALIVE_MAX_PINGS_WITHOUT_DATA` | `0` |  |
+| int | `ORCHESTRATOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS` | `0` |  |
+| int | `ORCHESTRATOR_GRPC_KEEPALIVE_TIMEOUT_MS` | `20000` |  |
+| int | `ORCHESTRATOR_GRPC_KEEPALIVE_TIME_MS` | `60000` |  |
 | string | `ORCHESTRATOR_HOST` | nil |  |
 | bool | `ORCHESTRATOR_MTLS_ENABLED` | nil |  |
 | string | `ORCHESTRATOR_PORT` | nil |  |