Merge branch 'stable-24-3' into merge/24-3/acceleration

Author: serbel324

.github/actions/s3cmd/action.yml

@@ -8,16 +8,16 @@ inputs:
     required: true
     description: "s3 key secret"
   s3_bucket:
-    required: true
+    required: false
     description: "s3 bucket"
   s3_endpoint:
     required: true
     description: "s3 endpoint"
   folder_prefix:
-    required: true
+    required: false
     description: "folder prefix"
   build_preset:
-    required: true
+    required: false
     description: "build preset like relwithdebinfo"
 runs:
   using: "composite"
@@ -35,7 +35,14 @@ runs:
         host_base = storage.yandexcloud.net
         host_bucket = %(bucket)s.storage.yandexcloud.net
         EOF
-        
+      env:
+        s3_key_id: ${{ inputs.s3_key_id }}
+        s3_secret_access_key: ${{ inputs.s3_key_secret }}
+
+    - name: export s3 path variables
+      shell: bash
+      if: inputs.build_preset
+      run: |
         folder="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
         
         BUILD_PRESET="${{ inputs.build_preset }}"
@@ -57,7 +64,4 @@ runs:
         echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
-        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
-      env:
-        s3_key_id: ${{ inputs.s3_key_id }}
-        s3_secret_access_key: ${{ inputs.s3_key_secret }}
+        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV

.github/config/muted_ya.txt

@@ -8,28 +8,30 @@ ydb/core/external_sources *
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
+ydb/core/kqp/ut/join KqpJoinOrder.Chain65Nodes
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
 ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
-ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_ResultCountAll_FilterL
-ydb/core/kqp/ut/olap KqpOlapWrite.WriteDeleteCleanGC
 ydb/core/kqp/ut/pg KqpPg.CreateIndex
+ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssueInteractiveTx+withSink
+ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssue+withSink
+ydb/core/kqp/ut/tx KqpSnapshotRead.ReadOnlyTxWithIndexCommitsOnConcurrentWrite+withSink
+ydb/core/kqp/ut/tx KqpSinkTx.InvalidateOnError
 ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
-ydb/core/kqp/ut/scheme KqpOlapScheme.DropTable
 ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
 ydb/core/kqp/ut/scheme [14/50]*
 ydb/core/kqp/ut/scheme [15/50]*
 ydb/core/kqp/ut/scheme [44/50]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
 ydb/core/kqp/ut/service [38/50]*
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/persqueue/ut [37/40] chunk chunk
 ydb/core/persqueue/ut [38/40] chunk chunk
 ydb/core/persqueue/ut TPQTest.*DirectRead*
@@ -96,7 +98,6 @@ ydb/tests/fq/yds *
 ydb/tests/fq/control_plane_storage *
 ydb/tests/functional/audit *
 ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication*
-ydb/tests/functional/clickbench test.py.test_plans[column]
 ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
 ydb/tests/functional/restarts test_restarts.py.*
@@ -106,4 +107,6 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*
 ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
 ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
+ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
+ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
 ydb/tests/functional/rename [test_rename.py */10] chunk chunk

.github/workflows/nightly_build.yml

@@ -0,0 +1,54 @@
+name: Nightly-Build  # workflow used to upload built binaries to s3
+on:
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        type: string
+        default: "auto-provisioned"
+        description: "runner label"
+      commit_sha:
+        type: string
+        default: ""
+jobs:
+  build_and_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        build_preset: ["relwithdebinfo", "release-asan"]
+    runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
+    name: Build and test ${{ matrix.build_preset }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.commit_sha }}
+        fetch-depth: 2
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
+    - name: Build and test
+      uses: ./.github/actions/build_and_test_ya
+      with:
+        build_preset: ${{ matrix.build_preset }}
+        build_target: "ydb/apps/ydbd"
+        increment: false
+        run_tests: false
+        put_build_results_to_cache: false
+        secs: ${{ format('{{"TESTMO_TOKEN2":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}',
+          secrets.TESTMO_TOKEN2, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
+        vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
+          vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}
+    - name: Setup s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: "ydb-builds"
+        s3_endpoint: ${{ vars.AWS_ENDPOINT }}
+        s3_key_id: ${{ secrets.AWS_KEY_ID }}
+        s3_key_secret: ${{ secrets.AWS_KEY_VALUE }}
+    
+    - name: sync results to s3 and publish links
+      shell: bash
+      run: |
+        set -x
+        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "ydb/apps/ydbd/ydbd" "s3://ydb-builds/${{ github.ref_name }}/${{ matrix.build_preset }}/ydbd" -d

.gitignore

@@ -7,6 +7,8 @@
 !*/
 # Unignore all files inside canondata dir
 !**/canondata/**
+# Allow docker files
+!Dockerfile
 
 /canonization_show_res.log
 
@@ -26,6 +28,9 @@ __pycache__/
 *.pb.h
 *.pb.cc
 
+# Other generated
+*.fbs.h
+
 # MacOS specific
 .DS_Store
 
@@ -40,6 +45,7 @@ __pycache__/
 .idea/
 .vscode/
 .clangd
+.antlr/
 
 # KDevelop IDE
 *.kdev4
@@ -79,3 +85,6 @@ list_result.log
 bin/config.json
 
 .vs/
+
+# handy for local junk, which is not intended to appear in the repo
+junk/

library/cpp/threading/future/core/future-inl.h

@@ -116,6 +116,9 @@ namespace NThreading {
             bool HasException() const {
                 return AtomicGet(State) == ExceptionSet;
             }
+            bool IsReady() const {
+                return AtomicGet(State) != NotReady;
+            }
 
             const T& GetValue(TDuration timeout = TDuration::Zero()) const {
                 AccessValue(timeout, ValueRead);
@@ -297,6 +300,9 @@ namespace NThreading {
             bool HasException() const {
                 return AtomicGet(State) == ExceptionSet;
             }
+            bool IsReady() const {
+                return AtomicGet(State) != NotReady;
+            }
 
             void GetValue(TDuration timeout = TDuration::Zero()) const {
                 TAtomicBase state = AtomicGet(State);
@@ -583,6 +589,10 @@ namespace NThreading {
     inline bool TFuture<T>::HasException() const {
         return State && State->HasException();
     }
+    template <typename T>
+    inline bool TFuture<T>::IsReady() const {
+        return State && State->IsReady();
+    }
 
     template <typename T>
     inline void TFuture<T>::Wait() const {
@@ -688,6 +698,9 @@ namespace NThreading {
     inline bool TFuture<void>::HasException() const {
         return State && State->HasException();
     }
+    inline bool TFuture<void>::IsReady() const {
+        return State && State->IsReady();
+    }
 
     inline void TFuture<void>::Wait() const {
         EnsureInitialized();
@@ -823,6 +836,11 @@ namespace NThreading {
         return State && State->HasException();
     }
 
+    template <typename T>
+    inline bool TPromise<T>::IsReady() const {
+        return State && State->IsReady();
+    }
+
     template <typename T>
     inline void TPromise<T>::SetException(const TString& e) {
         EnsureInitialized();
@@ -904,6 +922,10 @@ namespace NThreading {
         return State && State->HasException();
     }
 
+    inline bool TPromise<void>::IsReady() const {
+        return State && State->IsReady();
+    }
+
     inline void TPromise<void>::SetException(const TString& e) {
         EnsureInitialized();
         State->SetException(std::make_exception_ptr(yexception() << e));

library/cpp/threading/future/core/future.h

@@ -98,6 +98,12 @@ namespace NThreading {
         void TryRethrow() const;
         bool HasException() const;
 
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   NOTE: returns true even if value was extracted from promise
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
+
         void Wait() const;
         bool Wait(TDuration timeout) const;
         bool Wait(TInstant deadline) const;
@@ -153,6 +159,11 @@ namespace NThreading {
         void TryRethrow() const;
         bool HasException() const;
 
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
+
         void Wait() const;
         bool Wait(TDuration timeout) const;
         bool Wait(TInstant deadline) const;
@@ -216,6 +227,12 @@ namespace NThreading {
 
         void TryRethrow() const;
         bool HasException() const;
+
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   NOTE: returns true even if value was extracted from promise
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
         void SetException(const TString& e);
         void SetException(std::exception_ptr e);
         bool TrySetException(std::exception_ptr e);
@@ -256,6 +273,11 @@ namespace NThreading {
 
         void TryRethrow() const;
         bool HasException() const;
+
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
         void SetException(const TString& e);
         void SetException(std::exception_ptr e);
         bool TrySetException(std::exception_ptr e);

library/cpp/threading/future/future_ut.cpp

@@ -105,6 +105,7 @@ namespace {
 
             future = MakeFuture(345);
             UNIT_ASSERT(future.HasValue());
+            UNIT_ASSERT(future.IsReady());
             UNIT_ASSERT_EQUAL(future.GetValue(), 345);
         }
 
@@ -115,6 +116,7 @@ namespace {
 
             TFuture<void> future = promise.GetFuture();
             UNIT_ASSERT(future.HasValue());
+            UNIT_ASSERT(future.IsReady());
 
             future = MakeFuture();
             UNIT_ASSERT(future.HasValue());
@@ -523,6 +525,7 @@ namespace {
         {
             auto future1 = MakeErrorFuture<void>(std::make_exception_ptr(TFutureException()));
             UNIT_ASSERT(future1.HasException());
+            UNIT_ASSERT(future1.IsReady());
             UNIT_CHECK_GENERATED_EXCEPTION(future1.GetValue(), TFutureException);
 
             auto future2 = MakeErrorFuture<int>(std::make_exception_ptr(TFutureException()));
@@ -563,6 +566,7 @@ namespace {
             promise2.SetException("foo-exception");
             wait.Wait();
             UNIT_ASSERT(future2.HasException());
+            UNIT_ASSERT(!future1.IsReady());
             UNIT_ASSERT(!future1.HasValue() && !future1.HasException());
         }