viewer healthcheck pathmerge stable 24 3 11 hotfix

.github/CODEOWNERS

@@ -1,25 +1 @@
-/*.md @ydb-platform/docs
-
-/ydb/core/fq/ @ydb-platform/fq
-/ydb/core/public_http/ @ydb-platform/fq
-
-/ydb/docs/ @ydb-platform/docs
-
-/ydb/library/yql/ @ydb-platform/yql
-/ydb/library/yql/dq @ydb-platform/yql @ydb-platform/qp
-/ydb/library/yql/providers/generic @ydb-platform/fq
-/ydb/library/yql/providers/pq @ydb-platform/fq
-/ydb/library/yql/providers/s3 @ydb-platform/fq
-/ydb/library/yql/providers/solomon @ydb-platform/fq
-
-/ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
-
-/ydb/services/fq/ @ydb-platform/fq
-
-/ydb/core/kafka_proxy @ydb-platform/Topics
-/ydb/core/persqueue @ydb-platform/Topics
-/ydb/services/datastreams @ydb-platform/Topics
-/ydb/services/deprecated/persqueue_v0 @ydb-platform/Topics
-/ydb/services/persqueue_v1 @ydb-platform/Topics
-
-/ydb/core/config/ut @ydb-platform/core
+* @ydb-platform/ReleaseApprovers

.github/actions/s3cmd/action.yml

@@ -8,16 +8,16 @@ inputs:
     required: true
     description: "s3 key secret"
   s3_bucket:
-    required: true
+    required: false
     description: "s3 bucket"
   s3_endpoint:
     required: true
     description: "s3 endpoint"
   folder_prefix:
-    required: true
+    required: false
     description: "folder prefix"
   build_preset:
-    required: true
+    required: false
     description: "build preset like relwithdebinfo"
 runs:
   using: "composite"
@@ -35,7 +35,14 @@ runs:
         host_base = storage.yandexcloud.net
         host_bucket = %(bucket)s.storage.yandexcloud.net
         EOF
-
+      env:
+        s3_key_id: ${{ inputs.s3_key_id }}
+        s3_secret_access_key: ${{ inputs.s3_key_secret }}
+
+    - name: export s3 path variables
+      shell: bash
+      if: inputs.build_preset
+      run: |
         folder="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
 
         BUILD_PRESET="${{ inputs.build_preset }}"
@@ -57,7 +64,4 @@ runs:
         echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
-        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
-      env:
-        s3_key_id: ${{ inputs.s3_key_id }}
-        s3_secret_access_key: ${{ inputs.s3_key_secret }}
+        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV

.github/config/muted_ya.txt

@@ -8,21 +8,17 @@ ydb/core/external_sources *
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
-ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesActualization
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
-ydb/core/kqp/ut/olap KqpOlap.ScanQueryOltpAndOlap
 ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
-ydb/core/kqp/ut/olap KqpOlap.YqlScriptOltpAndOlap
-ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_ResultCountAll_FilterL
 ydb/core/kqp/ut/pg KqpPg.CreateIndex
+ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssueInteractiveTx+withSink
+ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssue+withSink
+ydb/core/kqp/ut/tx KqpSnapshotRead.ReadOnlyTxWithIndexCommitsOnConcurrentWrite+withSink
+ydb/core/kqp/ut/tx KqpSinkTx.InvalidateOnError
 ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
-ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
-ydb/core/kqp/ut/scheme KqpOlapScheme.DropTable
 ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
 ydb/core/kqp/ut/scheme [14/50]*
@@ -32,6 +28,8 @@ ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
 ydb/core/kqp/ut/service [38/50]*
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/persqueue/ut [37/40] chunk chunk
 ydb/core/persqueue/ut [38/40] chunk chunk
 ydb/core/persqueue/ut TPQTest.*DirectRead*
@@ -107,4 +105,6 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
 ydb/tests/functional/tenants test_tenants.py.*
 ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
 ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
+ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
+ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
 ydb/tests/functional/rename [test_rename.py */10] chunk chunk

.github/workflows/nightly_build.yml

@@ -0,0 +1,54 @@
+name: Nightly-Build  # workflow used to upload built binaries to s3
+on:
+  workflow_dispatch:
+    inputs:
+      runner_label:
+        type: string
+        default: "auto-provisioned"
+        description: "runner label"
+      commit_sha:
+        type: string
+        default: ""
+jobs:
+  build_and_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        build_preset: ["relwithdebinfo", "release-asan"]
+    runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
+    name: Build and test ${{ matrix.build_preset }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.commit_sha }}
+        fetch-depth: 2
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
+    - name: Build and test
+      uses: ./.github/actions/build_and_test_ya
+      with:
+        build_preset: ${{ matrix.build_preset }}
+        build_target: "ydb/apps/ydbd"
+        increment: false
+        run_tests: false
+        put_build_results_to_cache: false
+        secs: ${{ format('{{"TESTMO_TOKEN2":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}',
+          secrets.TESTMO_TOKEN2, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
+        vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
+          vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}
+    - name: Setup s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: "ydb-builds"
+        s3_endpoint: ${{ vars.AWS_ENDPOINT }}
+        s3_key_id: ${{ secrets.AWS_KEY_ID }}
+        s3_key_secret: ${{ secrets.AWS_KEY_VALUE }}
+
+    - name: sync results to s3 and publish links
+      shell: bash
+      run: |
+        set -x
+        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "ydb/apps/ydbd/ydbd" "s3://ydb-builds/${{ github.ref_name }}/${{ matrix.build_preset }}/ydbd" -d

library/cpp/lwtrace/mon/mon_lwtrace.cpp

@@ -301,7 +301,7 @@ struct TLogQuery {
             }
         } catch (...) {
             ythrow yexception()
-                << CurrentExceptionMessage()
+                << EncodeHtmlPcdata(CurrentExceptionMessage())
                 << " while parsing track log query: "
                 << Text;
         }
@@ -1853,7 +1853,7 @@ class TTracesHtmlPrinter {
         try {
             Os << src->GetStartTime().ToStringUpToSeconds();
         } catch (...) {
-            Os << "error: " << CurrentExceptionMessage();
+            Os << "error: " << EncodeHtmlPcdata(CurrentExceptionMessage());
         }
         Os << "</td>"
            << "<td><div class=\"dropdown\">"
@@ -3821,17 +3821,17 @@ class TLWTraceMonPage : public NMonitoring::IMonPage {
             }
         } catch (TPageGenBase& gen) {
             out.Clear();
-            out << gen.what();
+            out << EncodeHtmlPcdata(gen.what());
         } catch (...) {
             out.Clear();
             if (request.GetParams().Get("error") == "text") {
                 // Text error reply is helpful for ajax requests
                 out << NMonitoring::HTTPOKTEXT;
-                out << CurrentExceptionMessage();
+                out << EncodeHtmlPcdata(CurrentExceptionMessage());
             } else {
                 WWW_HTML(out) {
                     out << "<h2>Error</h2><pre>"
-                        << CurrentExceptionMessage()
+                        << EncodeHtmlPcdata(CurrentExceptionMessage())
                         << Endl;
                 }
             }

ydb/apps/version/version_definition.cpp

@@ -2,10 +2,21 @@
 
 NKikimrConfig::TCurrentCompatibilityInfo NKikimr::TCompatibilityInfo::MakeCurrent() {
     using TCurrentConstructor = NKikimr::TCompatibilityInfo::TProtoConstructor::TCurrentCompatibilityInfo;
-    // using TVersionConstructor = NKikimr::TCompatibilityInfo::TProtoConstructor::TVersion;
-    // using TCompatibilityRuleConstructor = NKikimr::TCompatibilityInfo::TProtoConstructor::TCompatibilityRule;
+    using TVersionConstructor = NKikimr::TCompatibilityInfo::TProtoConstructor::TVersion;
+    using TCompatibilityRuleConstructor = NKikimr::TCompatibilityInfo::TProtoConstructor::TCompatibilityRule;
 
     return TCurrentConstructor{
         .Application = "ydb",
+        .Version = TVersionConstructor{
+            .Year = 24,
+            .Major = 3,
+        },
+        .CanConnectTo = {
+            TCompatibilityRuleConstructor{
+                .Application = "nbs",
+                .LowerLimit = TVersionConstructor{ .Year = 23, .Major = 3 },
+                .UpperLimit = TVersionConstructor{ .Year = 24, .Major = 3 },
+            }
+        }
     }.ToPB();
-}
+}

ydb/apps/ydbd/ya.make

@@ -1,6 +1,8 @@
 PROGRAM(ydbd)
 
-NO_EXPORT_DYNAMIC_SYMBOLS()
+IF (NOT SANITIZER_TYPE)  # for some reasons some tests with asan are failed, see comment in CPPCOM-32
+    NO_EXPORT_DYNAMIC_SYMBOLS()
+ENDIF()
 
 IF (OS_LINUX)
     ALLOCATOR(TCMALLOC_256K)

ydb/core/audit/audit_log.h

@@ -1,19 +1,12 @@
 #pragma once
 
-#include <ydb/core/base/events.h>
+#include <utility>
+#include <atomic>
 
-#include <ydb/library/actors/core/actor.h>
-#include <ydb/library/actors/core/events.h>
-#include <library/cpp/logger/backend.h>
-#include <ydb/core/protos/config.pb.h>
-#include <ydb/library/services/services.pb.h>
-
-#include <library/cpp/logger/record.h>
-#include <ydb/library/actors/core/hfunc.h>
-#include <ydb/library/actors/core/log.h>
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
 
-#include <util/generic/strbuf.h>
-#include <util/datetime/base.h>
+#include <ydb/library/actors/core/actor.h>
 
 #define AUDIT_LOG_S(sys, expr)                                                                                                  \
     do {                                                                                                                        \
@@ -24,7 +17,7 @@
         }                                                                                                                       \
     } while (0) /**/
 
-#define AUDIT_LOG(expr) AUDIT_LOG_S((TlsActivationContext->ExecutorThread.ActorSystem), expr)
+#define AUDIT_LOG(expr) AUDIT_LOG_S((::NActors::TlsActivationContext->ExecutorThread.ActorSystem), expr)
 
 #define AUDIT_PART_NO_COND(key, value) AUDIT_PART_COND(key, value, true)
 #define AUDIT_PART_COND(key, value, condition)                                                                                    \
@@ -37,90 +30,14 @@
 #define GET_AUDIT_PART_MACRO(_1, _2, _3, NAME,...) NAME
 #define AUDIT_PART(...) GET_AUDIT_PART_MACRO(__VA_ARGS__, AUDIT_PART_COND, AUDIT_PART_NO_COND)(__VA_ARGS__)
 
+namespace NActors {
+    class TActorSystem;
+}
+
 namespace NKikimr::NAudit {
 
 extern std::atomic<bool> AUDIT_LOG_ENABLED;
 
-struct TEvAuditLog
-{
-    //
-    // Events declaration
-    //
-
-    enum EEvents
-    {
-        EvBegin = EventSpaceBegin(TKikimrEvents::ES_YDB_AUDIT_LOG),
-
-        // Request actors
-        EvWriteAuditLog = EvBegin + 0,
-
-        EvEnd
-    };
-
-    static_assert(EvEnd <= EventSpaceEnd(TKikimrEvents::ES_YDB_AUDIT_LOG),
-                  "expected EvEnd <= EventSpaceEnd(TKikimrEvents::ES_YDB_AUDIT_LOG)");
-
-    struct TEvWriteAuditLog
-        : public NActors::TEventLocal<TEvWriteAuditLog, EvWriteAuditLog>
-    {
-        TInstant Time;
-        TVector<std::pair<TString, TString>> Parts;
-
-        TEvWriteAuditLog(TInstant time, TVector<std::pair<TString, TString>>&& parts)
-            : Time(time)
-            , Parts(std::move(parts))
-        {}
-    };
-};
-
-class TAuditLogActor final
-    : public TActor<TAuditLogActor>
-{
-private:
-    const TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> LogBackends;
-public:
-    TAuditLogActor(TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> logBackends)
-        : TActor(&TThis::StateWork)
-        , LogBackends(std::move(logBackends))
-    {
-    }
-
-    static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
-        return NKikimrServices::TActivity::AUDIT_WRITER_ACTOR;
-    }
-
-private:
-    STFUNC(StateWork);
-
-    void HandlePoisonPill(
-        const TEvents::TEvPoisonPill::TPtr& ev,
-        const TActorContext& ctx);
-
-    void HandleWriteAuditLog(
-        const TEvAuditLog::TEvWriteAuditLog::TPtr& ev,
-        const TActorContext& ctx);
-
-    static void WriteLog(
-        const TString& log,
-        const TVector<THolder<TLogBackend>>& logBackends);
-
-    static TString GetJsonLog(
-        const TEvAuditLog::TEvWriteAuditLog::TPtr& ev);
-
-    static TString GetTxtLog(
-        const TEvAuditLog::TEvWriteAuditLog::TPtr& ev);
-
-    void HandleUnexpectedEvent(STFUNC_SIG);
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
 void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TString, TString>>&& parts);
 
-inline NActors::TActorId MakeAuditServiceID() {
-    return NActors::TActorId(0, TStringBuf("YDB_AUDIT"));
-}
-
-THolder<NActors::IActor> CreateAuditWriter(TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> logBackends);
-
 }   // namespace NKikimr::NAudit