From 6bc257b0afbf3d4c2893b0aa4f38f50feb5da4cc Mon Sep 17 00:00:00 2001 From: Bulat Gayazov Date: Wed, 4 Jun 2025 20:41:53 +0000 Subject: [PATCH] Added workflow for import SDK from ydb repo --- .github/import_generation.txt | 1 + .github/last_commit.txt | 1 + .github/scripts/copy_sources.sh | 93 +++++++++++++++++++++++++++++ .github/workflows/examples.yaml | 4 +- .github/workflows/import.yaml | 85 ++++++++++++++++++++++++++ .github/workflows/tests.yaml | 8 +-- .github/workflows/warmup_cache.yaml | 3 +- .gitignore | 3 +- 8 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 .github/import_generation.txt create mode 100644 .github/last_commit.txt create mode 100755 .github/scripts/copy_sources.sh create mode 100644 .github/workflows/import.yaml diff --git a/.github/import_generation.txt b/.github/import_generation.txt new file mode 100644 index 0000000000..3cacc0b93c --- /dev/null +++ b/.github/import_generation.txt @@ -0,0 +1 @@ +12 \ No newline at end of file diff --git a/.github/last_commit.txt b/.github/last_commit.txt new file mode 100644 index 0000000000..52036a2cda --- /dev/null +++ b/.github/last_commit.txt @@ -0,0 +1 @@ +ec36c2f7151e1e7718c2f0c0b2292383cab01e64 \ No newline at end of file diff --git a/.github/scripts/copy_sources.sh b/.github/scripts/copy_sources.sh new file mode 100755 index 0000000000..c5dce6b98d --- /dev/null +++ b/.github/scripts/copy_sources.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +tmp_dir=$(mktemp -d) + +echo "Copying sources..." + +cp -r $1/ydb/public/sdk/cpp/* $tmp_dir +echo "tmp_dir: $tmp_dir" + +rm -r $tmp_dir/client +rm -r $tmp_dir/src/client/cms +rm -r $tmp_dir/src/client/config +rm -r $tmp_dir/src/client/debug +rm -r $tmp_dir/src/client/draft + +rm -r $tmp_dir/include/ydb-cpp-sdk/client/cms +rm -r $tmp_dir/include/ydb-cpp-sdk/client/config +rm -r $tmp_dir/include/ydb-cpp-sdk/client/debug +rm -r $tmp_dir/include/ydb-cpp-sdk/client/draft + +rm -r $tmp_dir/tests/unit/client/draft + +mkdir -p $tmp_dir/src/api/client/yc_private +mkdir -p $tmp_dir/src/api/client/yc_public + +cp -r $1/ydb/public/api/client/yc_private/iam $tmp_dir/src/api/client/yc_private +cp -r $1/ydb/public/api/client/yc_private/operation $tmp_dir/src/api/client/yc_private +cp -r $1/ydb/public/api/client/yc_public/common $tmp_dir/src/api/client/yc_public +cp -r $1/ydb/public/api/client/yc_public/iam $tmp_dir/src/api/client/yc_public +cp -r $1/ydb/public/api/grpc $tmp_dir/src/api +cp -r $1/ydb/public/api/protos $tmp_dir/src/api + +rm -r $tmp_dir/src/api/protos/out +rm $tmp_dir/include/ydb-cpp-sdk/type_switcher.h $tmp_dir/src/version.h + +cp -r $2/util $tmp_dir +cp -r $2/library $tmp_dir + +cp -r $2/.devcontainer $tmp_dir +cp -r $2/.git $tmp_dir +cp -r $2/.github $tmp_dir +cp -r $2/contrib $tmp_dir +cp -r $2/cmake $tmp_dir +cp -r $2/scripts $tmp_dir +cp -r $2/third_party $tmp_dir +cp -r $2/tools $tmp_dir + +cp $2/.gitignore $tmp_dir +cp $2/.gitmodules $tmp_dir +cp $2/CMakePresets.json $tmp_dir +cp $2/CMakeLists.txt $tmp_dir +cp $2/LICENSE $tmp_dir +cp $2/README.md $tmp_dir + +cp $2/include/ydb-cpp-sdk/type_switcher.h $tmp_dir/include/ydb-cpp-sdk/type_switcher.h +cp $2/src/version.h $tmp_dir/src/version.h + +cd $2 + +find src/ include/ tests/ examples/ -type f -name "CMakeLists.txt" | while read f; +do + mkdir -p "$(dirname "$tmp_dir/$f")" && cp -p $f $tmp_dir/$f +done + +cd - + +echo "Copying completed" +echo "Patching sources..." + +rm -rf $tmp_dir/adapters $tmp_dir/client + +SED_INCLUDE='(^\s*#include\s*)(<|\")' + +find $tmp_dir -type f -regex ".*\(h\|cpp\|cpp.in\|c\|ipp\|jnj\|rl6\|h.txt\|proto\)$" | while read f; +do + sed -i -E \ + 's/'$SED_INCLUDE'ydb\/public\/api\//\1\2src\/api\//g; + s/(^\s*import\s*)(\")ydb\/public\/api\//\1\2src\/api\//g; + s/'$SED_INCLUDE'ydb\/public\/sdk\/cpp\/include\//\1\2/g; + s/'$SED_INCLUDE'ydb\/public\/sdk\/cpp\//\1\2/g; + s/'$SED_INCLUDE'library\/cpp\/retry\/retry_policy\.h/\1\2ydb-cpp-sdk\/library\/retry\/retry_policy\.h/g; + s/'$SED_INCLUDE'library\/cpp\/string_utils\/base64\/base64\.h/\1\2src\/library\/string_utils\/base64\/base64\.h/g; + s/(inline\s*)Dev/\1V3/g; + s/(inline\s*namespace\s*)Dev/\1V3/g;' $f +done + +echo "Patching completed" +echo "RSync..." + +rsync -c -I -W -r --delete --filter '- **/ya.make' --filter '- sdk_common.inc' $tmp_dir/ $2 +rm -rf $tmp_dir + +echo "RSync completed" diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 41170adfb5..08920ddf7b 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -35,13 +35,13 @@ jobs: options: '-h localhost' steps: - name: Checkout PR - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' with: submodules: true diff --git a/.github/workflows/import.yaml b/.github/workflows/import.yaml new file mode 100644 index 0000000000..b4fc76c5c8 --- /dev/null +++ b/.github/workflows/import.yaml @@ -0,0 +1,85 @@ +name: Import + +on: + workflow_dispatch: + +jobs: + import-ydb-sdk: + runs-on: ubuntu-latest + steps: + - name: Checkout ydb-cpp-sdk + uses: actions/checkout@v4 + with: + submodules: true + path: ydb-cpp-sdk + + - name: Checkout ydb + uses: actions/checkout@v4 + with: + repository: ydb-platform/ydb + ref: main + fetch-depth: 0 + submodules: true + path: ydb + + - name: Sync commits + run: | + sudo apt-get -y update + sudo apt-get -y install rsync + + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + + LAST_COMMIT=$(cat ydb-cpp-sdk/.github/last_commit.txt) + + COMMITS=$(git -C ydb log --format="%H" --reverse $LAST_COMMIT..main -- ydb/public/sdk/cpp ydb/public/api) + if [ -z "$COMMITS" ]; then + echo "No new commits to import" + exit 0 + fi + + GENERATION=$(cat ydb-cpp-sdk/.github/import_generation.txt) + + git -C ydb-cpp-sdk checkout -b import-pr-$GENERATION + for commit in $COMMITS; do + echo "Importing commit: $commit" + git -C ydb checkout $commit + ydb-cpp-sdk/.github/scripts/copy_sources.sh ydb ydb-cpp-sdk + if [ -z "$(git -C ydb-cpp-sdk status --porcelain)" ]; then + echo "No changes for commit: $commit" + continue + fi + echo $commit > ydb-cpp-sdk/.github/last_commit.txt + + message=$(git -C ydb show -s --format='%s') + + git -C ydb-cpp-sdk add . + git -C ydb-cpp-sdk status + + git -C ydb-cpp-sdk commit --author="$(git -C ydb show -s --format='%an <%ae>' $commit)" -m "$message" --no-edit + + git -C ydb-cpp-sdk show -s --pretty=medium + done + + - name: Create Pull Request + run: | + cd ydb-cpp-sdk + sudo apt-get -y install gh + + GENERATION=$(cat .github/import_generation.txt) + + echo "$((GENERATION + 1))" > .github/import_generation.txt + + git add .github/import_generation.txt + git commit -m "Update import generation: $GENERATION" + + git remote add token-origin https://${{ secrets.GITHUB_TOKEN }}@github.com/ydb-platform/ydb-cpp-sdk.git + git push token-origin import-pr-$GENERATION + + gh pr create \ + --base ${{ github.ref_name }} \ + --head import-pr-$GENERATION \ + --title "Import YDB C++ SDK $GENERATION" \ + --body "Automatic import of new commits from ydb repository" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1ba010d6d6..a8ac3eb337 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -24,13 +24,13 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout PR - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' with: submodules: true @@ -84,13 +84,13 @@ jobs: options: '-h localhost' steps: - name: Checkout PR - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' with: submodules: true diff --git a/.github/workflows/warmup_cache.yaml b/.github/workflows/warmup_cache.yaml index 04ed1d8ce4..4f8a062786 100644 --- a/.github/workflows/warmup_cache.yaml +++ b/.github/workflows/warmup_cache.yaml @@ -14,8 +14,7 @@ jobs: compiler: [clang, gcc] steps: - name: Checkout - uses: actions/checkout@v3 - if: github.event.pull_request.head.sha == '' + uses: actions/checkout@v4 with: submodules: true - name: Install dependencies diff --git a/.gitignore b/.gitignore index ae8e52d38c..9e762ad8e7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,8 @@ _deps !*/ # Unignore all files inside canondata dir !*/canondata/* - +# Unignore all files inside .github dir +!*/.github/* # C libraries *.so *.a