protobuf 5 compatibility

.github/workflows/tests.yaml

@@ -17,14 +17,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8]
-        environment: [py, py-tls, py-proto3, py-tls-proto3]
-        folder: [ydb, tests]
+        python-version: [3.8, 3.9]
+        environment: [py-proto5, py-tls-proto5, py-proto4, py-tls-proto4, py-proto3, py-tls-proto3]
+        folder: [ydb, tests --ignore=tests/topics, tests/topics]
         exclude:
-          - environment: py-tls
+          - environment: py-tls-proto5
+            folder: ydb
+          - environment: py-tls-proto4
             folder: ydb
           - environment: py-tls-proto3
             folder: ydb
+          - environment: py-tls-proto5
+            folder: tests/topics
+          - environment: py-tls-proto4
+            folder: tests/topics
+          - environment: py-tls-proto3
+            folder: tests/topics
 
     steps:
     - uses: actions/checkout@v1

Makefile

@@ -1,9 +1,13 @@
-protobuf: protobuf-3 protobuf-4
+protobuf: protobuf-3 protobuf-4 protobuf-5
 
 protobuf-3:
 	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-3 --build-arg GRPCIO_VER=1.39.0 --build-arg PY_PROTOBUF_VER=3.20.3
 	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-3 python generate_protoc.py --target-version=v3
 
 protobuf-4:
 	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-4
-	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-4 python generate_protoc.py
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-4 python generate_protoc.py --target-version=v4
+
+protobuf-5:
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-5 --build-arg PY_PROTOBUF_VER=5.26.1 --build-arg PROTOC_VER=26.1
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-5 python generate_protoc.py --target-version=v5

generate-protobuf.Dockerfile

@@ -9,7 +9,7 @@ RUN \
     python -m pip install grpcio-tools==${GRPCIO_VER} && \
     python -m pip install protobuf==${PY_PROTOBUF_VER}
 
-ENV PROTOC_VER=21.8
+ARG PROTOC_VER=21.8
 RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip && \
     unzip protoc-*.zip && \
     rm -f protoc-*.zip && \

requirements.txt

@@ -1,4 +1,4 @@
 grpcio>=1.42.0
 packaging
-protobuf>=3.13.0,<5.0.0
+protobuf>=3.13.0,<6.0.0
 aiohttp<4

test-requirements.txt

@@ -11,15 +11,15 @@ docker==5.0.0
 docker-compose==1.29.2
 dockerpty==0.4.1
 docopt==0.6.2
-grpcio==1.42.0
+grpcio>=1.42.0
 idna==3.2
 importlib-metadata==4.6.1
 iniconfig==1.1.1
 jsonschema==3.2.0
 packaging==21.0
 paramiko==2.10.1
 pluggy==0.13.1
-protobuf>=3.13.0,<5.0.0
+protobuf>=3.13.0,<6.0.0
 py==1.10.0
 pycparser==2.20
 PyNaCl==1.4.0

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py,py-proto3,py-tls,py-tls-proto3,style,pylint,black,protoc,py-cov
+envlist = py-proto5,py-proto4,py-proto3,py-tls-proto5,py-tls-proto4,py-tls-proto3,style,pylint,black,protoc,py-cov-proto4
 minversion = 4.2.6
 skipsdist = True
 ignore_basepython_conflict = true
@@ -12,35 +12,67 @@ setenv =
 deps =
     -r{toxinidir}/test-requirements.txt
 
-[testenv:dev]
+[testenv:dev-proto5]
 commands =
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<6.0.0
+
+[testenv:dev-proto4]
+commands =
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<5.0.0
 
 [testenv:dev-proto3]
 commands =
 deps =
     -r{toxinidir}/test-requirements.txt
     protobuf<4.0.0
 
-[testenv:py]
+[testenv:py-proto5]
 commands =
-    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
+    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} --ignore=tests/topics
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<6.0.0
 
-[testenv:py-cov]
+[testenv:py-proto4]
+commands =
+    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} --ignore=tests/topics
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<5.0.0
+
+[testenv:py-cov-proto4]
 commands =
     pytest -v -m "not tls" \
         --cov-report html:cov_html --cov=ydb \
         --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<5.0.0
 
 [testenv:py-proto3]
 commands =
-    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
+    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} --ignore=tests/topics
 deps =
     -r{toxinidir}/test-requirements.txt
     protobuf<4.0.0
 
-[testenv:py-tls]
+[testenv:py-tls-proto5]
+commands =
+    pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs}
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<6.0.0
+
+[testenv:py-tls-proto4]
 commands =
     pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs}
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<5.0.0
 
 [testenv:py-tls-proto3]
 commands =
@@ -52,12 +84,12 @@ deps =
 [testenv:black-format]
 skip_install = true
 commands =
-    black ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4"
+    black ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4|ydb/_grpc/v5"
 
 [testenv:black]
 skip_install = true
 commands =
-    black --diff --check ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4"
+    black --diff --check ydb examples tests --extend-exclude "ydb/_grpc/v3|ydb/_grpc/v4|ydb/_grpc/v5"
 
 [testenv:pylint]
 deps = pylint

ydb/_grpc/common/__init__.py

@@ -36,11 +36,20 @@
         from ydb._grpc.v3 import protos  # noqa
 
         sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v3.protos"]
-    else:
+    elif protobuf_version < Version("5.0"):
         from ydb._grpc.v4 import *  # noqa
 
         sys.modules["ydb._grpc.common"] = sys.modules["ydb._grpc.v4"]
 
         from ydb._grpc.v4 import protos  # noqa
 
         sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v4.protos"]
+
+    else:
+        from ydb._grpc.v5 import *  # noqa
+
+        sys.modules["ydb._grpc.common"] = sys.modules["ydb._grpc.v5"]
+
+        from ydb._grpc.v5 import protos  # noqa
+
+        sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v5.protos"]