Enable kafka port for local ydb container

.github/CODEOWNERS

@@ -1,25 +1 @@
-/*.md @ydb-platform/docs
-
-/ydb/core/fq/ @ydb-platform/fq
-/ydb/core/public_http/ @ydb-platform/fq
-
-/ydb/docs/ @ydb-platform/docs
-
-/ydb/library/yql/ @ydb-platform/yql
-/ydb/library/yql/dq @ydb-platform/yql @ydb-platform/qp
-/ydb/library/yql/providers/generic @ydb-platform/fq
-/ydb/library/yql/providers/pq @ydb-platform/fq
-/ydb/library/yql/providers/s3 @ydb-platform/fq
-/ydb/library/yql/providers/solomon @ydb-platform/fq
-
-/ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
-
-/ydb/services/fq/ @ydb-platform/fq
-
-/ydb/core/kafka_proxy @ydb-platform/Topics
-/ydb/core/persqueue @ydb-platform/Topics
-/ydb/services/datastreams @ydb-platform/Topics
-/ydb/services/deprecated/persqueue_v0 @ydb-platform/Topics
-/ydb/services/persqueue_v1 @ydb-platform/Topics
-
-/ydb/core/config/ut @ydb-platform/core
+* @ydb-platform/ReleaseApprovers

.github/actions/s3cmd/action.yml

@@ -8,16 +8,16 @@ inputs:
     required: true
     description: "s3 key secret"
   s3_bucket:
-    required: true
+    required: false
     description: "s3 bucket"
   s3_endpoint:
     required: true
     description: "s3 endpoint"
   folder_prefix:
-    required: true
+    required: false
     description: "folder prefix"
   build_preset:
-    required: true
+    required: false
     description: "build preset like relwithdebinfo"
 runs:
   using: "composite"
@@ -35,7 +35,14 @@ runs:
         host_base = storage.yandexcloud.net
         host_bucket = %(bucket)s.storage.yandexcloud.net
         EOF
-
+      env:
+        s3_key_id: ${{ inputs.s3_key_id }}
+        s3_secret_access_key: ${{ inputs.s3_key_secret }}
+
+    - name: export s3 path variables
+      shell: bash
+      if: inputs.build_preset
+      run: |
         folder="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
 
         BUILD_PRESET="${{ inputs.build_preset }}"
@@ -46,7 +53,7 @@ runs:
           debug)
             folder+="-debug"
             ;;
-          release-*)
+          release*)
             folder+="-${BUILD_PRESET/release-/}"
             ;;
           *)
@@ -57,7 +64,4 @@ runs:
         echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
-        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
-      env:
-        s3_key_id: ${{ inputs.s3_key_id }}
-        s3_secret_access_key: ${{ inputs.s3_key_secret }}
+        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV

.github/actions/test_ya/action.yml

@@ -103,7 +103,7 @@ runs:
           debug)
             TESTMO_SOURCE="ya-${ARCH}-debug"
             ;;
-          release-*)
+          release*)
             TESTMO_SOURCE="ya-${ARCH}-${BUILD_PRESET/release-/}"
             ;;
           *)
@@ -180,6 +180,9 @@ runs:
           relwithdebinfo)
             params+=(--build "relwithdebinfo")
             ;;
+          release)
+            params+=(--build "release")
+            ;;
           release-asan)
             params+=(
               --build "release" --sanitize="address"

.github/config/muted_ya.txt

@@ -8,30 +8,30 @@ ydb/core/external_sources *
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
-ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesActualization
+ydb/core/kqp/ut/join KqpJoinOrder.Chain65Nodes
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
-ydb/core/kqp/ut/olap KqpOlap.ScanQueryOltpAndOlap
 ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
-ydb/core/kqp/ut/olap KqpOlap.YqlScriptOltpAndOlap
-ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_ResultCountAll_FilterL
 ydb/core/kqp/ut/pg KqpPg.CreateIndex
+ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssueInteractiveTx+withSink
+ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssue+withSink
+ydb/core/kqp/ut/tx KqpSnapshotRead.ReadOnlyTxWithIndexCommitsOnConcurrentWrite+withSink
+ydb/core/kqp/ut/tx KqpSinkTx.InvalidateOnError
 ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
-ydb/core/kqp/ut/scheme KqpOlapScheme.DropTable
 ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
 ydb/core/kqp/ut/scheme [14/50]*
 ydb/core/kqp/ut/scheme [15/50]*
 ydb/core/kqp/ut/scheme [44/50]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
 ydb/core/kqp/ut/service [38/50]*
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/persqueue/ut [37/40] chunk chunk
 ydb/core/persqueue/ut [38/40] chunk chunk
 ydb/core/persqueue/ut TPQTest.*DirectRead*
@@ -107,4 +107,6 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*
 ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
 ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
+ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
+ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
 ydb/tests/functional/rename [test_rename.py */10] chunk chunk

.github/docker/Dockerfile

@@ -44,6 +44,7 @@ COPY --from=builder \
 EXPOSE ${GRPC_TLS_PORT:-2135}
 EXPOSE ${GRPC_PORT:-2136}
 EXPOSE ${MON_PORT:-8765}
+EXPOSE ${YDB_KAFKA_PROXY_PORT:-9092}
 
 HEALTHCHECK --start-period=60s --interval=1s CMD sh ./health_check
 

.github/docker/files/initialize_local_ydb

@@ -8,6 +8,7 @@ export YDB_GRPC_ENABLE_TLS="true"
 export GRPC_TLS_PORT=${GRPC_TLS_PORT:-2135}
 export GRPC_PORT=${GRPC_PORT:-2136}
 export YDB_GRPC_TLS_DATA_PATH="/ydb_certs"
+export YDB_KAFKA_PROXY_PORT=${YDB_KAFKA_PROXY_PORT:-9092}
 
 /local_ydb deploy --ydb-working-dir /ydb_data --ydb-binary-path /ydbd --fixed-ports --dont-use-log-files;
 

.github/workflows/docs_release.yaml

@@ -5,33 +5,83 @@ on:
     branches:
       - 'main'
       - 'stable-**'
-    paths: 
+    paths:
       - 'ydb/docs/**'
   workflow_dispatch:
 
 jobs:
-  release:
+  build:
+    runs-on: ubuntu-latest
     concurrency:
       group: release-documentation-${{ github.ref }}
       cancel-in-progress: true
-    runs-on: ubuntu-latest
+    permissions: write-all
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Extract version
-        shell: bash
-        run: echo "version=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | sed -e 's|stable-|v|g' -e 's|-|.|g' >> $GITHUB_OUTPUT
-        id: extract_version
-      - name: Release
-        uses: diplodoc-platform/docs-release-action@v1
+        uses: actions/checkout@v4
+
+      - name: Build
+        uses: diplodoc-platform/docs-build-action@v3
         with:
           revision: "${{ github.sha }}"
-          version: "${{ steps.extract_version.outputs.version }}"
-          project-name: ${{ secrets.DOCS_PROJECT_NAME }}
           src-root: ${{ vars.SRC_ROOT }}
-          storage-bucket: ${{ secrets.DOCS_AWS_BUCKET }}
+
+  upload:
+    needs: build
+    runs-on: ubuntu-latest
+    concurrency:
+      group: release-documentation-${{ github.ref }}
+      cancel-in-progress: true
+    permissions: write-all
+    steps:
+      - name: Upload
+        uses: diplodoc-platform/docs-upload-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          storage-bucket: "common/${{ secrets.DOCS_PROJECT_NAME }}"
           storage-endpoint: ${{ vars.DOCS_AWS_ENDPOINT }}
           storage-access-key-id: ${{ secrets.DOCS_AWS_KEY_ID }}
           storage-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
           storage-region: ${{ vars.DOCS_AWS_REGION }}
-          shared-storage-bucket: true
+
+  release:
+    needs: upload
+    runs-on: ubuntu-latest
+    concurrency:
+      group: release-documentation-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Extract version # turn branch name into version string
+        id: extract_version
+        shell: bash
+        run: |
+          echo "version=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" \
+            | sed -e 's|stable-|v|g' -e 's|-|.|g' \
+            >> $GITHUB_OUTPUT
+
+      - name: Get default branch name
+        id: get_default
+        run: |
+          default=$(curl -fsS \
+            https://raw.githubusercontent.com/ydb-platform/ydb/refs/heads/main/ydb/docs/default-branch.txt)
+          echo "DEFAULT_BRANCH=$default" >> $GITHUB_ENV
+
+      - name: Set update-only-version # only skip default-branch commits
+        id: set-update-only-version
+        run: |
+          current=${GITHUB_REF##*/}    # e.g. 'main' or 'stable-X'
+          if [[ "$current" == "$DEFAULT_BRANCH" ]]; then
+            echo "update_only_version=false" >> $GITHUB_ENV
+          else
+            echo "update_only_version=true"  >> $GITHUB_ENV
+          fi
+
+      - name: Release
+        uses: diplodoc-platform/docs-release-action@v2
+        with:
+          revision: "${{ github.sha }}"
+          version:  "${{ steps.extract_version.outputs.version }}"
+          storage-bucket: ${{ secrets.DOCS_PROJECT_NAME }}
+          storage-access-key-id: ${{ secrets.DOCS_AWS_KEY_ID }}
+          storage-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+          update-only-version: "${{ env.update_only_version }}"

.github/workflows/nightly_build.yml

@@ -0,0 +1,54 @@
+name: Nightly-Build  # workflow used to upload built binaries to s3
+on:
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        type: string
+        default: "auto-provisioned"
+        description: "runner label"
+      commit_sha:
+        type: string
+        default: ""
+jobs:
+  build_and_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        build_preset: ["release", "relwithdebinfo", "release-asan"]
+    runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
+    name: Build and test ${{ matrix.build_preset }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.commit_sha }}
+        fetch-depth: 2
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
+    - name: Build and test
+      uses: ./.github/actions/build_and_test_ya
+      with:
+        build_preset: ${{ matrix.build_preset }}
+        build_target: "ydb/apps/ydbd"
+        increment: false
+        run_tests: false
+        put_build_results_to_cache: false
+        secs: ${{ format('{{"TESTMO_TOKEN2":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}',
+          secrets.TESTMO_TOKEN2, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
+        vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
+          vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}
+    - name: Setup s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: "ydb-builds"
+        s3_endpoint: ${{ vars.AWS_ENDPOINT }}
+        s3_key_id: ${{ secrets.AWS_KEY_ID }}
+        s3_key_secret: ${{ secrets.AWS_KEY_VALUE }}
+
+    - name: sync results to s3 and publish links
+      shell: bash
+      run: |
+        set -x
+        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "ydb/apps/ydbd/ydbd" "s3://ydb-builds/${{ github.ref_name }}/${{ matrix.build_preset }}/ydbd" -d

.github/workflows/pr_check.yml

@@ -203,7 +203,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build_preset: ["relwithdebinfo", "release-asan", "release-clang14"]
+        build_preset: ["relwithdebinfo", "release-asan"]
     runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
     name: Build and test ${{ matrix.build_preset }}
     steps:
@@ -247,7 +247,7 @@ jobs:
             https://api.github.com/repos/${{github.repository}}/commits/${{github.event.pull_request.head.sha}}/status | \
             jq -cr '.statuses | .[] | select(.state=="success") | select(.context | startswith("build_")) | .context' | \
             wc -l )
-          if [[ $successbuilds == "3" ]];then
+          if [[ $successbuilds == "2" ]];then
             integrated_status="success"
           else
             integrated_status="failure"

.gitignore

@@ -7,6 +7,8 @@
 !*/
 # Unignore all files inside canondata dir
 !**/canondata/**
+# Allow docker files
+!Dockerfile
 
 /canonization_show_res.log
 
@@ -26,6 +28,9 @@ __pycache__/
 *.pb.h
 *.pb.cc
 
+# Other generated
+*.fbs.h
+
 # MacOS specific
 .DS_Store
 
@@ -40,6 +45,7 @@ __pycache__/
 .idea/
 .vscode/
 .clangd
+.antlr/
 
 # KDevelop IDE
 *.kdev4
@@ -79,3 +85,6 @@ list_result.log
 bin/config.json
 
 .vs/
+
+# handy for local junk, which is not intended to appear in the repo
+junk/