From 93e6833c7447eca30bbd720adc43803eb92dc079 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov Date: Sun, 20 Apr 2025 18:43:16 +0300 Subject: [PATCH 1/4] Add devcontainer configuration for YDB Python SDK - Create Dockerfile for building the SDK environment - Add docker-compose configuration for YDB services - Include YDB configuration file for storage and channel profiles - Set up initialization and configuration scripts for YDB CLI - Configure Prometheus for monitoring YDB services --- .devcontainer/Dockerfile | 8 +++++ .devcontainer/commands/initialize.sh | 5 +++ .devcontainer/commands/postCreate.sh | 22 ++++++++++++ .devcontainer/commands/postStart.sh | 8 +++++ .devcontainer/compose.yml | 51 ++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 54 ++++++++++++++++++++++++++++ 6 files changed, 148 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100755 .devcontainer/commands/initialize.sh create mode 100755 .devcontainer/commands/postCreate.sh create mode 100755 .devcontainer/commands/postStart.sh create mode 100644 .devcontainer/compose.yml create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..723c1843 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,8 @@ +FROM mcr.microsoft.com/devcontainers/python:3.12-bookworm + +# [Optional] Uncomment if you want to install more tools +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment if you want to install ydb cli +RUN curl -fsSL https://raw.githubusercontent.com/ydb-platform/ydb/refs/heads/main/ydb/apps/ydb/install/install.sh | bash diff --git a/.devcontainer/commands/initialize.sh b/.devcontainer/commands/initialize.sh new file mode 100755 index 00000000..9be0294d --- /dev/null +++ b/.devcontainer/commands/initialize.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +git config --local user.email "$(git config user.email)" +git config --local user.name "$(git config user.name)" diff --git a/.devcontainer/commands/postCreate.sh b/.devcontainer/commands/postCreate.sh new file mode 100755 index 00000000..f05b6be9 --- /dev/null +++ b/.devcontainer/commands/postCreate.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + + +# Set up YDB profile if ydb cli exists +if which ydb > /dev/null 2>&1; then + ENDPOINT=$(echo ${YDB_CONNECTION_STRING_SECURE:-$YDB_CONNECTION_STRING} | awk -F/ '{print $1 "//" $3}') + DATABASE=$(echo ${YDB_CONNECTION_STRING_SECURE:-$YDB_CONNECTION_STRING} | cut -d/ -f4-) + CA_FILE_OPTION="" + + if [ -n "$YDB_SSL_ROOT_CERTIFICATES_FILE" ]; then + ENDPOINT="${ENDPOINT/grpc:/grpcs:}" + CA_FILE_OPTION="--ca-file ${YDB_SSL_ROOT_CERTIFICATES_FILE}" + fi + + ydb config profile replace local \ + --endpoint "$ENDPOINT" \ + --database "/$DATABASE" \ + $CA_FILE_OPTION + + ydb config profile activate local +fi diff --git a/.devcontainer/commands/postStart.sh b/.devcontainer/commands/postStart.sh new file mode 100755 index 00000000..4ffe6fe3 --- /dev/null +++ b/.devcontainer/commands/postStart.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + + +# Install dependencies +pip install -r requirements.txt +# Install test dependencies +# pip install -r test-requirements.txt diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml new file mode 100644 index 00000000..56039007 --- /dev/null +++ b/.devcontainer/compose.yml @@ -0,0 +1,51 @@ +volumes: + ydb-data: + # driver: local + # driver_opts: + # type: tmpfs + # device: tmpfs + # o: size=80g + ydb-certs: + +services: + sdk: + build: + context: . + dockerfile: Dockerfile + hostname: sdk + + volumes: + - ydb-certs:/ydb_certs + - ../:/workspaces/ydb-python-sdk:cached + + environment: + - YDB_IMAGE_VERSION=24.3 + - YDB_CREDENTIALS_USER=root + - YDB_CREDENTIALS_PASSWORD=1234 + - YDB_CONNECTION_STRING=grpc://ydb:2136/local + - YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local + - YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + ydb: + image: ghcr.io/ydb-platform/local-ydb:24.3 + restart: unless-stopped + hostname: ydb + platform: linux/amd64 + + ports: + - 2135:2135 + - 2136:2136 + - 8765:8765 + + volumes: + - ydb-data:/ydb_data + - ydb-certs:/ydb_certs + + environment: + - YDB_USE_IN_MEMORY_PDISKS=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..836e0f3c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,54 @@ +{ + "name": "Python & YDB", + "service": "sdk", + "dockerComposeFile": "compose.yml", + "workspaceFolder": "/workspaces/ydb-python-sdk", + // Allows the container to use ptrace, which is useful for debugging. + "capAdd": [ + "SYS_PTRACE" + ], + // Disables seccomp, which can be necessary for some debugging tools to function correctly. + "securityOpt": [ + "seccomp=unconfined" + ], + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/git": {}, + "ghcr.io/devcontainers/features/common-utils": {}, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 2135, + 2136, + 8765, + 9090, + 9464 + ], + // Use 'initializeCommand' to run commands before the container is created. + "initializeCommand": "chmod +x .devcontainer/commands/initialize.sh && .devcontainer/commands/initialize.sh", + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "chmod +x .devcontainer/commands/postCreate.sh && .devcontainer/commands/postCreate.sh", + // Use 'postStartCommand' to run commands after the container is started. + "postStartCommand": "chmod +x .devcontainer/commands/postStart.sh && .devcontainer/commands/postStart.sh", + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "ms-python.autopep8", + "ms-python.debugpy", + "ms-python.flake8", + "ms-python.isort", + "ms-python.pylint", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.vscode-python-envs" + ] + } + }, + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + "remoteUser": "root", + "mounts": [ + "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,readonly" + ] +} From 31f837681d1db6c94c017009afce943cedfe5eea Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov Date: Tue, 22 Apr 2025 12:52:58 +0300 Subject: [PATCH 2/4] Update devcontainer mount to use specific SSH signing key Signed-off-by: Vladislav Polyakov --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 836e0f3c..21e43676 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -49,6 +49,6 @@ // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. "remoteUser": "root", "mounts": [ - "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,readonly" + "source=${localEnv:HOME}/.ssh/id_ed25519_signing,target=/root/.ssh/id_ed25519_signing,type=bind,readonly" ] } From 54abfd2207fa54f79d1754dcff48eefb9bdc9599 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov Date: Tue, 22 Apr 2025 14:50:13 +0300 Subject: [PATCH 3/4] Add sign-off configuration and GPG settings in initialization scripts Signed-off-by: Vladislav Polyakov --- .devcontainer/commands/initialize.sh | 3 ++- .devcontainer/commands/postCreate.sh | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.devcontainer/commands/initialize.sh b/.devcontainer/commands/initialize.sh index 9be0294d..926031b6 100755 --- a/.devcontainer/commands/initialize.sh +++ b/.devcontainer/commands/initialize.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e -git config --local user.email "$(git config user.email)" +git config --local format.signoff true git config --local user.name "$(git config user.name)" +git config --local user.email "$(git config user.email)" diff --git a/.devcontainer/commands/postCreate.sh b/.devcontainer/commands/postCreate.sh index f05b6be9..40078bf5 100755 --- a/.devcontainer/commands/postCreate.sh +++ b/.devcontainer/commands/postCreate.sh @@ -1,6 +1,11 @@ #!/bin/bash set -e +if git config --get commit.gpgsign | grep -q true; then + git config --global gpg.format ssh + git config --global gpg.ssh.defaultKeyCommand 'ssh-add -L' + git config --global gpg.ssh.allowedSigners '~/.ssh/allowed_signers' +fi # Set up YDB profile if ydb cli exists if which ydb > /dev/null 2>&1; then From e0b703b379fde5ee3ce4f9731e963109e6d15534 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Wed, 23 Apr 2025 18:53:01 +0300 Subject: [PATCH 4/4] Update postStart.sh --- .devcontainer/commands/postStart.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.devcontainer/commands/postStart.sh b/.devcontainer/commands/postStart.sh index 4ffe6fe3..bf38ac35 100755 --- a/.devcontainer/commands/postStart.sh +++ b/.devcontainer/commands/postStart.sh @@ -4,5 +4,8 @@ set -e # Install dependencies pip install -r requirements.txt -# Install test dependencies -# pip install -r test-requirements.txt +# Install ydb package +pip install -e . +# Install tox for CI +pip install tox +