From c4827ffc70b7baecc3e4c304de531d34bd02e0cb Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 28 Jan 2025 15:17:51 +0000 Subject: [PATCH 01/65] start --- ydb/core/tablet_flat/flat_cxx_database.h | 1 + ydb/core/tx/schemeshard/schemeshard__init.cpp | 9 +++++++++ ydb/core/tx/schemeshard/schemeshard_import.cpp | 11 +++++++++++ ydb/core/tx/schemeshard/schemeshard_info_types.h | 1 + ydb/core/tx/schemeshard/schemeshard_schema.h | 1 + 5 files changed, 23 insertions(+) diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index e6ea695ba436..64352d1b9105 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -238,6 +238,7 @@ template <> struct NSchemeTypeMapper { typedef i32 Ty template <> struct NSchemeTypeMapper { typedef i64 Type; }; template <> struct NSchemeTypeMapper { typedef i64 Type; }; template <> struct NSchemeTypeMapper { typedef i64 Type; }; +template <> struct NSchemeTypeMapper { typedef TVector Type; }; /// only for compatibility with old code template diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index a6f26ad50798..6b1732916ccc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4464,6 +4464,15 @@ struct TSchemeShard::TTxInit : public TTransactionBase { item.Metadata = NBackup::TMetadata::Deserialize(rowset.GetValue()); } + if (rowset.HaveValue()) { + const ui64 count = rowset.GetValue().size(); + TVector changefeeds(count); + for (ui64 i = 0; i < count; ++i) { + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(changefeeds[i], rowset.GetValue())[i]); + } + item.Changefeeds = changefeeds; + } + item.State = static_cast(rowset.GetValue()); item.WaitTxId = rowset.GetValueOrDefault(InvalidTxId); item.NextIndexIdx = rowset.GetValueOrDefault(0); diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index 04f51430691a..1bfee2bd7121 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -189,6 +189,17 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf db.Table().Key(importInfo->Id, itemIdx).Update( NIceDb::TUpdate(item.Metadata.Serialize()) ); + const ui64 count = item.Changefeeds.size(); + TVector jsonChangefeeds; + jsonChangefeeds.reserve(count); + + for (const auto& changefeed : item.Changefeeds) { + jsonChangefeeds.push_back(changefeed.SerializeAsString()); + } + + db.Table().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate(jsonChangefeeds) + ); } void TSchemeShard::PersistImportItemPreparedCreationQuery(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 42e45e06683a..fb36b66ee0e1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2851,6 +2851,7 @@ struct TImportInfo: public TSimpleRefCount { TMaybe PreparedCreationQuery; TMaybeFail Permissions; NBackup::TMetadata Metadata; + TVector Changefeeds; EState State = EState::GetScheme; ESubState SubState = ESubState::AllocateTxId; diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index ce4c4e26cb33..b92ff1533f4d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1562,6 +1562,7 @@ struct Schema : NIceDb::Schema { struct PreparedCreationQuery : Column<14, NScheme::NTypeIds::String> {}; struct Permissions : Column<11, NScheme::NTypeIds::String> {}; struct Metadata : Column<12, NScheme::NTypeIds::String> {}; + struct Changefeeds : Column<13, NScheme::NTypeIds::Json> {}; struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; From e9ee1784a1e5a5eb16cbbfd04a7644c3e911023e Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 28 Jan 2025 17:35:06 +0000 Subject: [PATCH 02/65] CreateChangefeedsPropose --- .../schemeshard_import_flow_proposals.cpp | 12 ++++++++++++ ydb/core/tx/schemeshard/schemeshard_info_types.h | 1 + 2 files changed, 13 insertions(+) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index b8fd66be70c6..e5a7a5b73d8c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -8,6 +8,18 @@ namespace NKikimr { namespace NSchemeShard { +void CreateChangefeedsPropose(THolder& propose, const TImportInfo::TItem& item) { + auto& record = propose->Record; + const auto& changefeeds = item.Changefeeds; + + for (const auto& changefeed : changefeeds) { + auto& modifyScheme = *record.AddTransaction(); + const auto& cdcStream = modifyScheme.MutableCreateCdcStream(); + cdcStream->MutableTableName(); + } + +} + THolder CreateTablePropose( TSchemeShard* ss, TTxId txId, diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index fb36b66ee0e1..ee078e97dede 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2850,6 +2850,7 @@ struct TImportInfo: public TSimpleRefCount { TString CreationQuery; TMaybe PreparedCreationQuery; TMaybeFail Permissions; + TVector Changefeeds; NBackup::TMetadata Metadata; TVector Changefeeds; From ad58c1894e732ddfcb0cf0a420bd48d56344f323 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 29 Jan 2025 09:36:39 +0000 Subject: [PATCH 03/65] CreateChangefeedsPropose --- .../schemeshard_import_flow_proposals.cpp | 14 +++++++--- .../schemeshard_import_scheme_getter.cpp | 6 ++++ ydb/core/ydb_convert/table_description.cpp | 28 +++++++++++++------ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index e5a7a5b73d8c..96a276211f39 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -8,16 +8,20 @@ namespace NKikimr { namespace NSchemeShard { -void CreateChangefeedsPropose(THolder& propose, const TImportInfo::TItem& item) { +bool CreateChangefeedsPropose(THolder& propose, const TImportInfo::TItem& item, TString& error) { auto& record = propose->Record; const auto& changefeeds = item.Changefeeds; for (const auto& changefeed : changefeeds) { auto& modifyScheme = *record.AddTransaction(); - const auto& cdcStream = modifyScheme.MutableCreateCdcStream(); - cdcStream->MutableTableName(); + auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); + Ydb::StatusIds::StatusCode status; + auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); + if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { + return false; + } } - + return true; } THolder CreateTablePropose( @@ -85,6 +89,8 @@ THolder CreateTablePropose( return nullptr; } + CreateChangefeedsPropose(propose, item, error); + return propose; } diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 84b740ad9947..b4424f3651c6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -51,6 +51,11 @@ class TSchemeGetter: public TActorBootstrapped { return errorType == S3Errors::RESOURCE_NOT_FOUND || errorType == S3Errors::NO_SUCH_KEY; } + static TString ChangefeedKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx, const TString& changefeedName) { + Y_ABORT_UNLESS(itemIdx < (ui32)settings.items_size()); + return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/" << changefeedName << "/changefeed_description.pb"; + } + void HeadObject(const TString& key) { auto request = Model::HeadObjectRequest() .WithKey(key); @@ -432,6 +437,7 @@ class TSchemeGetter: public TActorBootstrapped { const TString MetadataKey; TString SchemeKey; const TString PermissionsKey; + const TString ChangefeedKey; const ui32 Retries; ui32 Attempt = 0; diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 346f036950b5..f3719f4901df 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -1200,8 +1200,9 @@ void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, } } -bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, - const Ydb::Table::Changefeed& in, Ydb::StatusIds::StatusCode& status, TString& error) { +template +bool FillChangefeedDescriptionCommon(NKikimrSchemeOp::TCdcStreamDescription& out, + const T& in, Ydb::StatusIds::StatusCode& status, TString& error) { out.SetName(in.name()); out.SetVirtualTimestamps(in.virtual_timestamps()); @@ -1241,6 +1242,17 @@ bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, return false; } + for (const auto& [key, value] : in.attributes()) { + auto& attr = *out.AddUserAttributes(); + attr.SetKey(key); + attr.SetValue(value); + } + + return true; +} + +bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, + const Ydb::Table::Changefeed& in, Ydb::StatusIds::StatusCode& status, TString& error) { if (in.initial_scan()) { if (!AppData()->FeatureFlags.GetEnableChangefeedInitialScan()) { status = Ydb::StatusIds::UNSUPPORTED; @@ -1249,14 +1261,12 @@ bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, } out.SetState(NKikimrSchemeOp::ECdcStreamState::ECdcStreamStateScan); } + return FillChangefeedDescriptionCommon(out, in, status, error); +} - for (const auto& [key, value] : in.attributes()) { - auto& attr = *out.AddUserAttributes(); - attr.SetKey(key); - attr.SetValue(value); - } - - return true; +bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, + const Ydb::Table::ChangefeedDescription& in, Ydb::StatusIds::StatusCode& status, TString& error) { + return FillChangefeedDescriptionCommon(out, in, status, error); } void FillTableStats(Ydb::Table::DescribeTableResult& out, From 02791a111ef37fb14844be72de9009faa6aed0dc Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 29 Jan 2025 14:14:15 +0000 Subject: [PATCH 04/65] list obj --- .../schemeshard_import_scheme_getter.cpp | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index b4424f3651c6..6266b0c00eaf 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -56,6 +56,34 @@ class TSchemeGetter: public TActorBootstrapped { return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/" << changefeedName << "/changefeed_description.pb"; } + void ListObjects(const TString& prefix) { + auto request = Model::ListObjectsRequest() + .WithPrefix(prefix); + + Send(Client, new TEvExternalStorage::TEvListObjectsRequest(request)); + } + + void HandleChangefeeds(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { + const auto& result = ev.Get()->Get()->Result; + + LOG_D("HandleChangefeeds TEvExternalStorage::TEvListObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "ListObject")) { + return; + } + TString a; + + //Создать поле класса с ветором ключей (перед этим пофильтровать именно ченджфиды - пути до директорий) + //создать индекс уже скаченных + //сделать по аналогии с импортом + for (const auto& x : result.GetResult().GetContents()) { + x.GetKey(). + } + + } + void HeadObject(const TString& key) { auto request = Model::HeadObjectRequest() .WithKey(key); @@ -247,7 +275,7 @@ class TSchemeGetter: public TActorBootstrapped { item.Permissions = std::move(permissions); auto nextStep = [this]() { - Reply(); + StartDonloadingChangefeeds(); }; if (NeedValidateChecksums) { @@ -317,12 +345,20 @@ class TSchemeGetter: public TActorBootstrapped { TActor::PassAway(); } - void Download(const TString& key) { + void DownloadCommon() { if (Client) { Send(Client, new TEvents::TEvPoisonPill()); } Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); + } + + void DownloadWithoutKey() { + DownloadCommon(); + ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); + } + void Download(const TString& key) { + DownloadCommon(); HeadObject(key); } @@ -342,6 +378,10 @@ class TSchemeGetter: public TActorBootstrapped { Download(ChecksumKey); } + void DownloadChangefeeds() { + DownloadWithoutKey(); + } + void ResetRetries() { Attempt = 0; } @@ -358,6 +398,12 @@ class TSchemeGetter: public TActorBootstrapped { Become(&TThis::StateDownloadPermissions); } + void StartDonloadingChangefeeds() { + ResetRetries(); + DownloadChangefeeds(); + Become(&TThis::StateDownloadChangefeeds); + } + void StartValidatingChecksum(const TString& key, const TString& object, std::function checksumValidatedCallback) { ChecksumKey = NBackup::ChecksumKey(key); Checksum = NBackup::ComputeChecksum(object); @@ -418,6 +464,17 @@ class TSchemeGetter: public TActorBootstrapped { } } + STATEFN(StateDownloadChangefeeds) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvListObjectsResponse, HandleChangefeeds); + hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChangefeeds); + hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleChangefeeds); + + sFunc(TEvents::TEvWakeup, DownloadChangefeeds); + sFunc(TEvents::TEvPoisonPill, PassAway); + } + } + STATEFN(StateDownloadChecksum) { switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChecksum); From 953651e0c28b62fe293d1373d254fb0ca046c46d Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 30 Jan 2025 09:23:39 +0000 Subject: [PATCH 05/65] import_scheme_getter is complere --- .../schemeshard_import_flow_proposals.cpp | 3 +- .../schemeshard_import_scheme_getter.cpp | 146 ++++++++++++++++-- .../tx/schemeshard/schemeshard_info_types.h | 8 +- 3 files changed, 140 insertions(+), 17 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 96a276211f39..d66d4253da59 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -12,7 +12,7 @@ bool CreateChangefeedsPropose(THolderRecord; const auto& changefeeds = item.Changefeeds; - for (const auto& changefeed : changefeeds) { + for (const auto& [changefeed, topic]: changefeeds) { auto& modifyScheme = *record.AddTransaction(); auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); Ydb::StatusIds::StatusCode status; @@ -20,6 +20,7 @@ bool CreateChangefeedsPropose(THolder { return errorType == S3Errors::RESOURCE_NOT_FOUND || errorType == S3Errors::NO_SUCH_KEY; } - static TString ChangefeedKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx, const TString& changefeedName) { - Y_ABORT_UNLESS(itemIdx < (ui32)settings.items_size()); - return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/" << changefeedName << "/changefeed_description.pb"; + static TString ChangefeedDescriptionKey(const TString& changefeedPrefix) { + return TStringBuilder() << changefeedPrefix << "/changefeed_description.pb"; + } + + static TString TopicDescriptionKey(const TString& changefeedPrefix) { + return TStringBuilder() << changefeedPrefix << "/topic_description.pb"; } void ListObjects(const TString& prefix) { @@ -73,13 +76,21 @@ class TSchemeGetter: public TActorBootstrapped { if (!CheckResult(result, "ListObject")) { return; } - TString a; - //Создать поле класса с ветором ключей (перед этим пофильтровать именно ченджфиды - пути до директорий) - //создать индекс уже скаченных - //сделать по аналогии с импортом - for (const auto& x : result.GetResult().GetContents()) { - x.GetKey(). + const auto& objects = result.GetResult().GetContents(); + ChangefeedsKeys.reserve(objects.size()); + + for (const auto& obj : objects) { + const TFsPath& path = obj.GetKey(); + if (path.GetName() == "changefeed_description.pb") { + ChangefeedsKeys.push_back(path.Dirname()); + } + } + + if (!ChangefeedsKeys.empty()) { + HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[0])); + } else { + Reply(); } } @@ -161,6 +172,36 @@ class TSchemeGetter: public TActorBootstrapped { GetObject(ChecksumKey, std::make_pair(0, contentLength - 1)); } + void HandleChangefeeds(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + LOG_D("HandleChangefeeds TEvExternalStorage::TEvHeadObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "HeadObject")) { + return; + } + + const auto contentLength = result.GetResult().GetContentLength(); + GetObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), std::make_pair(0, contentLength - 1)); + } + + void HandleTopics(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + LOG_D("HandleChangefeeds TEvExternalStorage::TEvHeadObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "HeadObject")) { + return; + } + + const auto contentLength = result.GetResult().GetContentLength(); + GetObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), std::make_pair(0, contentLength - 1)); + } + void GetObject(const TString& key, const std::pair& range) { auto request = Model::GetObjectRequest() .WithKey(key) @@ -238,7 +279,7 @@ class TSchemeGetter: public TActorBootstrapped { if (NeedDownloadPermissions) { StartDownloadingPermissions(); } else { - Reply(); + StartDownloadingChangefeeds(); } }; @@ -275,7 +316,7 @@ class TSchemeGetter: public TActorBootstrapped { item.Permissions = std::move(permissions); auto nextStep = [this]() { - StartDonloadingChangefeeds(); + StartDownloadingChangefeeds(); }; if (NeedValidateChecksums) { @@ -307,6 +348,82 @@ class TSchemeGetter: public TActorBootstrapped { ChecksumValidatedCallback(); } + void HandleChangefeed(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + const auto& msg = *ev->Get(); + const auto& result = msg.Result; + + LOG_D("HandleChangefeeds TEvExternalStorage::TEvGetObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "GetObject")) { + return; + } + + Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size()); + auto& item = ImportInfo->Items.at(ItemIdx); + + LOG_T("Trying to parse changefeed" + << ": self# " << SelfId() + << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); + + Ydb::Table::ChangefeedDescription changefeed; + if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &changefeed)) { + return Reply(false, "Cannot parse permissions"); + } + item.Changefeeds[IndexDownloadedChangefeed].ChangefeedDescription = std::move(changefeed); + + auto nextStep = [this]() { + HeadObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); + }; + + if (NeedValidateChecksums) { + StartValidatingChecksum(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), msg.Body, nextStep); + } else { + nextStep(); + } + } + + void HandleTopic(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + const auto& msg = *ev->Get(); + const auto& result = msg.Result; + + LOG_D("HandleChangefeeds TEvExternalStorage::TEvGetObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "GetObject")) { + return; + } + + Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size()); + auto& item = ImportInfo->Items.at(ItemIdx); + + LOG_T("Trying to parse changefeed" + << ": self# " << SelfId() + << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); + + Ydb::Topic::DescribeTopicResult topic; + if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &topic)) { + return Reply(false, "Cannot parse permissions"); + } + item.Changefeeds[IndexDownloadedChangefeed].Topic = std::move(topic); + + auto nextStep = [this]() { + if (++IndexDownloadedChangefeed == ChangefeedsKeys.size()) { + Reply(); + } else { + HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); + } + }; + + if (NeedValidateChecksums) { + StartValidatingChecksum(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), msg.Body, nextStep); + } else { + nextStep(); + } + } + template bool CheckResult(const TResult& result, const TStringBuf marker) { if (result.IsSuccess()) { @@ -398,7 +515,7 @@ class TSchemeGetter: public TActorBootstrapped { Become(&TThis::StateDownloadPermissions); } - void StartDonloadingChangefeeds() { + void StartDownloadingChangefeeds() { ResetRetries(); DownloadChangefeeds(); Become(&TThis::StateDownloadChangefeeds); @@ -468,7 +585,7 @@ class TSchemeGetter: public TActorBootstrapped { switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvListObjectsResponse, HandleChangefeeds); hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChangefeeds); - hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleChangefeeds); + hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleChangefeed); sFunc(TEvents::TEvWakeup, DownloadChangefeeds); sFunc(TEvents::TEvPoisonPill, PassAway); @@ -494,7 +611,8 @@ class TSchemeGetter: public TActorBootstrapped { const TString MetadataKey; TString SchemeKey; const TString PermissionsKey; - const TString ChangefeedKey; + TVector ChangefeedsKeys; + ui64 IndexDownloadedChangefeed = 0; const ui32 Retries; ui32 Attempt = 0; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index ee078e97dede..0be807bab8fa 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2837,6 +2837,11 @@ struct TImportInfo: public TSimpleRefCount { S3 = 0, }; + struct TChangefeedImportDescriptions { + Ydb::Table::ChangefeedDescription ChangefeedDescription; + Ydb::Topic::DescribeTopicResult Topic; + }; + struct TItem { enum class ESubState: ui8 { AllocateTxId = 0, @@ -2850,9 +2855,8 @@ struct TImportInfo: public TSimpleRefCount { TString CreationQuery; TMaybe PreparedCreationQuery; TMaybeFail Permissions; - TVector Changefeeds; NBackup::TMetadata Metadata; - TVector Changefeeds; + TVector Changefeeds; EState State = EState::GetScheme; ESubState SubState = ESubState::AllocateTxId; From 5cfaac1eb7f54f56540ce0f0a80ee9e990e70167 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 30 Jan 2025 09:33:42 +0000 Subject: [PATCH 06/65] fix log --- .../tx/schemeshard/schemeshard_import_scheme_getter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index f77dbcb3f848..66be5081a3dd 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -172,7 +172,7 @@ class TSchemeGetter: public TActorBootstrapped { GetObject(ChecksumKey, std::make_pair(0, contentLength - 1)); } - void HandleChangefeeds(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + void HandleChangefeed(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; LOG_D("HandleChangefeeds TEvExternalStorage::TEvHeadObjectResponse" @@ -187,10 +187,10 @@ class TSchemeGetter: public TActorBootstrapped { GetObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), std::make_pair(0, contentLength - 1)); } - void HandleTopics(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + void HandleTopic(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - LOG_D("HandleChangefeeds TEvExternalStorage::TEvHeadObjectResponse" + LOG_D("HandleTopic TEvExternalStorage::TEvHeadObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -584,7 +584,7 @@ class TSchemeGetter: public TActorBootstrapped { STATEFN(StateDownloadChangefeeds) { switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvListObjectsResponse, HandleChangefeeds); - hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChangefeeds); + hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChangefeed); hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleChangefeed); sFunc(TEvents::TEvWakeup, DownloadChangefeeds); From 3844535510a7f0183b519991e5a5acdbdbb1fa32 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 30 Jan 2025 10:40:19 +0000 Subject: [PATCH 07/65] added topics --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 13 +++++++++---- ydb/core/tx/schemeshard/schemeshard_import.cpp | 8 +++++++- .../schemeshard_import_flow_proposals.cpp | 2 +- ydb/core/tx/schemeshard/schemeshard_schema.h | 1 + 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 6b1732916ccc..1f6e0789468d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4464,13 +4464,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase { item.Metadata = NBackup::TMetadata::Deserialize(rowset.GetValue()); } - if (rowset.HaveValue()) { + if (rowset.HaveValue() && rowset.HaveValue()) { const ui64 count = rowset.GetValue().size(); - TVector changefeeds(count); + TVector changefeeds; + changefeeds.reserve(count); for (ui64 i = 0; i < count; ++i) { - Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(changefeeds[i], rowset.GetValue())[i]); + Ydb::Table::ChangefeedDescription changefeed; + Ydb::Topic::DescribeTopicResult topic; + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(changefeed, rowset.GetValue())[i]); + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(topic, rowset.GetValue())[i]); + changefeeds.emplace_back(changefeed, topic); } - item.Changefeeds = changefeeds; + item.Changefeeds = std::move(changefeeds); } item.State = static_cast(rowset.GetValue()); diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index 1bfee2bd7121..de8694050c35 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -191,15 +191,21 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf ); const ui64 count = item.Changefeeds.size(); TVector jsonChangefeeds; + TVector jsonTopics; jsonChangefeeds.reserve(count); + jsonTopics.reserve(count); - for (const auto& changefeed : item.Changefeeds) { + for (const auto& [changefeed, topic] : item.Changefeeds) { jsonChangefeeds.push_back(changefeed.SerializeAsString()); + jsonTopics.push_back(topic.SerializeAsString()); } db.Table().Key(importInfo->Id, itemIdx).Update( NIceDb::TUpdate(jsonChangefeeds) ); + db.Table().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate(jsonTopics) + ); } void TSchemeShard::PersistImportItemPreparedCreationQuery(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index d66d4253da59..4f5ab09ad411 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -20,7 +20,7 @@ bool CreateChangefeedsPropose(THolder {}; struct Metadata : Column<12, NScheme::NTypeIds::String> {}; struct Changefeeds : Column<13, NScheme::NTypeIds::Json> {}; + struct Topics : Column<14, NScheme::NTypeIds::Json> {}; struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; From c371c983c91cdfeb3b18a101bf1068c049958146 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 30 Jan 2025 10:49:16 +0000 Subject: [PATCH 08/65] fix name --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 1f6e0789468d..4fc14b558ae7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4466,7 +4466,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase { if (rowset.HaveValue() && rowset.HaveValue()) { const ui64 count = rowset.GetValue().size(); - TVector changefeeds; + TVector changefeeds; changefeeds.reserve(count); for (ui64 i = 0; i < count; ++i) { Ydb::Table::ChangefeedDescription changefeed; From 078c578c041323e8cbe10e40f35ca466f866f41f Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 30 Jan 2025 14:53:14 +0300 Subject: [PATCH 09/65] try fix build --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 4 ++-- ydb/core/ydb_convert/table_description.h | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 4fc14b558ae7..62a745410dfe 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4471,8 +4471,8 @@ struct TSchemeShard::TTxInit : public TTransactionBase { for (ui64 i = 0; i < count; ++i) { Ydb::Table::ChangefeedDescription changefeed; Ydb::Topic::DescribeTopicResult topic; - Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(changefeed, rowset.GetValue())[i]); - Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(topic, rowset.GetValue())[i]); + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(changefeed, rowset.GetValue()[i] )); + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(topic, rowset.GetValue()[i] )); changefeeds.emplace_back(changefeed, topic); } item.Changefeeds = std::move(changefeeds); diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index e6a38df6b68d..ce33e9ca21a7 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -89,6 +89,8 @@ void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, // in bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, const Ydb::Table::Changefeed& in, Ydb::StatusIds::StatusCode& status, TString& error); +bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, + const Ydb::Table::ChangefeedDescription& in, Ydb::StatusIds::StatusCode& status, TString& error); // out void FillTableStats(Ydb::Table::DescribeTableResult& out, From d7a0c4b9b0768a7bf4ee12c2d13b18a2158ea2be Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 30 Jan 2025 15:55:56 +0300 Subject: [PATCH 10/65] fix logs --- .../schemeshard/schemeshard_import_scheme_getter.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 66be5081a3dd..b00f853cab09 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -175,7 +175,7 @@ class TSchemeGetter: public TActorBootstrapped { void HandleChangefeed(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - LOG_D("HandleChangefeeds TEvExternalStorage::TEvHeadObjectResponse" + LOG_D("HandleChangefeed TEvExternalStorage::TEvHeadObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -352,7 +352,7 @@ class TSchemeGetter: public TActorBootstrapped { const auto& msg = *ev->Get(); const auto& result = msg.Result; - LOG_D("HandleChangefeeds TEvExternalStorage::TEvGetObjectResponse" + LOG_D("HandleChangefeed TEvExternalStorage::TEvGetObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -369,7 +369,7 @@ class TSchemeGetter: public TActorBootstrapped { Ydb::Table::ChangefeedDescription changefeed; if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &changefeed)) { - return Reply(false, "Cannot parse permissions"); + return Reply(false, "Cannot parse сhangefeed"); } item.Changefeeds[IndexDownloadedChangefeed].ChangefeedDescription = std::move(changefeed); @@ -388,7 +388,7 @@ class TSchemeGetter: public TActorBootstrapped { const auto& msg = *ev->Get(); const auto& result = msg.Result; - LOG_D("HandleChangefeeds TEvExternalStorage::TEvGetObjectResponse" + LOG_D("HandleTopic TEvExternalStorage::TEvGetObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -399,13 +399,13 @@ class TSchemeGetter: public TActorBootstrapped { Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size()); auto& item = ImportInfo->Items.at(ItemIdx); - LOG_T("Trying to parse changefeed" + LOG_T("Trying to parse topic" << ": self# " << SelfId() << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); Ydb::Topic::DescribeTopicResult topic; if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &topic)) { - return Reply(false, "Cannot parse permissions"); + return Reply(false, "Cannot parse topic"); } item.Changefeeds[IndexDownloadedChangefeed].Topic = std::move(topic); From 4a7297ae20d0b8efbf89730de9febaddbb3df065 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 31 Jan 2025 16:04:56 +0300 Subject: [PATCH 11/65] fix review 1 --- ydb/core/protos/flat_scheme_op.proto | 6 ++ ydb/core/tablet_flat/flat_cxx_database.h | 1 - ydb/core/tx/schemeshard/schemeshard__init.cpp | 18 ++-- .../tx/schemeshard/schemeshard_import.cpp | 16 +--- .../schemeshard_import__create.cpp | 22 +++++ .../schemeshard_import_flow_proposals.cpp | 43 +++++---- .../schemeshard_import_flow_proposals.h | 6 ++ .../schemeshard_import_scheme_getter.cpp | 92 ++++++++++--------- .../tx/schemeshard/schemeshard_info_types.h | 2 +- ydb/core/tx/schemeshard/schemeshard_schema.h | 8 +- ydb/core/ydb_convert/table_description.h | 2 +- 11 files changed, 130 insertions(+), 86 deletions(-) diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 1b41719b37be..7c66beb28159 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -23,6 +23,7 @@ import "ydb/library/mkql_proto/protos/minikql.proto"; import "ydb/public/api/protos/ydb_coordination.proto"; import "ydb/public/api/protos/ydb_export.proto"; import "ydb/public/api/protos/ydb_table.proto"; +import "ydb/public/api/protos/ydb_topic.proto"; import "ydb/public/api/protos/ydb_value.proto"; import "google/protobuf/empty.proto"; @@ -2124,3 +2125,8 @@ message TBackupBackupCollection { optional NKikimrProto.TPathID PathId = 2; optional string TargetDir = 3; // must be set on Rewrite } + +message TImportTableChangefeeds { + repeated Ydb.Table.ChangefeedDescription ChangefeedDescriptions = 1; + repeated Ydb.Topic.DescribeTopicResult Topics = 2; +} diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 64352d1b9105..e6ea695ba436 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -238,7 +238,6 @@ template <> struct NSchemeTypeMapper { typedef i32 Ty template <> struct NSchemeTypeMapper { typedef i64 Type; }; template <> struct NSchemeTypeMapper { typedef i64 Type; }; template <> struct NSchemeTypeMapper { typedef i64 Type; }; -template <> struct NSchemeTypeMapper { typedef TVector Type; }; /// only for compatibility with old code template diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 62a745410dfe..a50c6e981c73 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4464,16 +4464,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase { item.Metadata = NBackup::TMetadata::Deserialize(rowset.GetValue()); } - if (rowset.HaveValue() && rowset.HaveValue()) { - const ui64 count = rowset.GetValue().size(); + if (rowset.HaveValue()) { + const auto& importTableChangefeeds = rowset.GetValue(); + const int count = importTableChangefeeds.GetChangefeedDescriptions().size(); + Y_ASSERT(importTableChangefeeds.GetTopics().size() == count); + TVector changefeeds; changefeeds.reserve(count); - for (ui64 i = 0; i < count; ++i) { - Ydb::Table::ChangefeedDescription changefeed; - Ydb::Topic::DescribeTopicResult topic; - Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(changefeed, rowset.GetValue()[i] )); - Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(topic, rowset.GetValue()[i] )); - changefeeds.emplace_back(changefeed, topic); + + for (int i = 0; i < count; ++i) { + const Ydb::Table::ChangefeedDescription& changefeedImport = importTableChangefeeds.GetChangefeedDescriptions()[i]; + const Ydb::Topic::DescribeTopicResult& topicImport = importTableChangefeeds.GetTopics()[i]; + changefeeds.emplace_back(changefeedImport, topicImport); } item.Changefeeds = std::move(changefeeds); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index de8694050c35..d0fbc37cf47f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -189,22 +189,16 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf db.Table().Key(importInfo->Id, itemIdx).Update( NIceDb::TUpdate(item.Metadata.Serialize()) ); - const ui64 count = item.Changefeeds.size(); - TVector jsonChangefeeds; - TVector jsonTopics; - jsonChangefeeds.reserve(count); - jsonTopics.reserve(count); + + NKikimrSchemeOp::TImportTableChangefeeds changefeeds; for (const auto& [changefeed, topic] : item.Changefeeds) { - jsonChangefeeds.push_back(changefeed.SerializeAsString()); - jsonTopics.push_back(topic.SerializeAsString()); + *changefeeds.MutableChangefeedDescriptions()->Add() = changefeed; + *changefeeds.MutableTopics()->Add() = topic; } db.Table().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate(jsonChangefeeds) - ); - db.Table().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate(jsonTopics) + NIceDb::TUpdate(changefeeds) ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index c3d9a639966f..6088e15423f7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -371,6 +371,28 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(propose); Send(Self->SelfId(), std::move(propose)); + + CreateChangefeeds(importInfo, itemIdx, txId); + } + + void CreateChangefeeds(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { + Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::Proposed; + + LOG_I("TImport::TTxProgress: CreateChangefeeds propose" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); + + auto proposes = CreateChangefeedsProposes(Self, txId, item); + + for (auto& propose : proposes) { + Send(Self->SelfId(), std::move(propose)); + } } void ExecutePreparedQuery(TTransactionContext& txc, TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 4f5ab09ad411..c078a7bd0d58 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -8,23 +8,6 @@ namespace NKikimr { namespace NSchemeShard { -bool CreateChangefeedsPropose(THolder& propose, const TImportInfo::TItem& item, TString& error) { - auto& record = propose->Record; - const auto& changefeeds = item.Changefeeds; - - for (const auto& [changefeed, topic]: changefeeds) { - auto& modifyScheme = *record.AddTransaction(); - auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); - Ydb::StatusIds::StatusCode status; - auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); - if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { - return false; - } - cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); - } - return true; -} - THolder CreateTablePropose( TSchemeShard* ss, TTxId txId, @@ -90,8 +73,6 @@ THolder CreateTablePropose( return nullptr; } - CreateChangefeedsPropose(propose, item, error); - return propose; } @@ -105,6 +86,30 @@ THolder CreateTablePropose( return CreateTablePropose(ss, txId, importInfo, itemIdx, unused); } +TVector> CreateChangefeedsProposes( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item) { + const auto& changefeeds = item.Changefeeds; + TVector> result; + result.reserve(changefeeds.size()); + + for (const auto& [changefeed, topic]: changefeeds) { + auto propose = MakeHolder(ui64(txId), ss->TabletID()); + auto& record = propose->Record; + auto& modifyScheme = *record.AddTransaction(); + auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); + + TString error; + Ydb::StatusIds::StatusCode status; + + auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); + + Y_ABORT_UNLESS(FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)); + + cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); + result.push_back(std::move(propose)); + } + return result; +} + static NKikimrSchemeOp::TTableDescription GetTableDescription(TSchemeShard* ss, const TPathId& pathId) { auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId); auto record = desc->GetRecord(); diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h index a99221db35b4..b2b50a6fa94e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h @@ -20,6 +20,12 @@ THolder CreateTablePropose( ui32 itemIdx ); +TVector> CreateChangefeedsProposes( + TSchemeShard* ss, + TTxId txId, + const TImportInfo::TItem& item +); + THolder RestorePropose( TSchemeShard* ss, TTxId txId, diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index b00f853cab09..be9e28bba9c1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -59,42 +59,6 @@ class TSchemeGetter: public TActorBootstrapped { return TStringBuilder() << changefeedPrefix << "/topic_description.pb"; } - void ListObjects(const TString& prefix) { - auto request = Model::ListObjectsRequest() - .WithPrefix(prefix); - - Send(Client, new TEvExternalStorage::TEvListObjectsRequest(request)); - } - - void HandleChangefeeds(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { - const auto& result = ev.Get()->Get()->Result; - - LOG_D("HandleChangefeeds TEvExternalStorage::TEvListObjectResponse" - << ": self# " << SelfId() - << ", result# " << result); - - if (!CheckResult(result, "ListObject")) { - return; - } - - const auto& objects = result.GetResult().GetContents(); - ChangefeedsKeys.reserve(objects.size()); - - for (const auto& obj : objects) { - const TFsPath& path = obj.GetKey(); - if (path.GetName() == "changefeed_description.pb") { - ChangefeedsKeys.push_back(path.Dirname()); - } - } - - if (!ChangefeedsKeys.empty()) { - HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[0])); - } else { - Reply(); - } - - } - void HeadObject(const TString& key) { auto request = Model::HeadObjectRequest() .WithKey(key); @@ -424,6 +388,42 @@ class TSchemeGetter: public TActorBootstrapped { } } + void ListObjects(const TString& prefix) { + auto request = Model::ListObjectsRequest() + .WithPrefix(prefix); + + Send(Client, new TEvExternalStorage::TEvListObjectsRequest(request)); + } + + void HandleChangefeeds(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { + const auto& result = ev.Get()->Get()->Result; + + LOG_D("HandleChangefeeds TEvExternalStorage::TEvListObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "ListObject")) { + return; + } + + const auto& objects = result.GetResult().GetContents(); + ChangefeedsKeys.reserve(objects.size()); + + for (const auto& obj : objects) { + const TFsPath& path = obj.GetKey(); + if (path.GetName() == "changefeed_description.pb") { + ChangefeedsKeys.push_back(path.Dirname()); + } + } + + if (!ChangefeedsKeys.empty()) { + HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); + } else { + Reply(); + } + + } + template bool CheckResult(const TResult& result, const TStringBuf marker) { if (result.IsSuccess()) { @@ -462,20 +462,20 @@ class TSchemeGetter: public TActorBootstrapped { TActor::PassAway(); } - void DownloadCommon() { + void CreateClient() { if (Client) { Send(Client, new TEvents::TEvPoisonPill()); } Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); } - void DownloadWithoutKey() { - DownloadCommon(); + void ListingChangefeeds() { + CreateClient(); ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); } void Download(const TString& key) { - DownloadCommon(); + CreateClient(); HeadObject(key); } @@ -496,7 +496,7 @@ class TSchemeGetter: public TActorBootstrapped { } void DownloadChangefeeds() { - DownloadWithoutKey(); + ListingChangefeeds(); } void ResetRetries() { @@ -592,6 +592,16 @@ class TSchemeGetter: public TActorBootstrapped { } } + STATEFN(StateDownloadTopics) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleTopic); + hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleTopic); + + sFunc(TEvents::TEvWakeup, DownloadChangefeeds); + sFunc(TEvents::TEvPoisonPill, PassAway); + } + } + STATEFN(StateDownloadChecksum) { switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChecksum); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 0be807bab8fa..6bc4f28052eb 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2884,7 +2884,7 @@ struct TImportInfo: public TSimpleRefCount { static bool IsDone(const TItem& item); }; - + ui64 Id; // TxId from the original TEvCreateImportRequest TString Uid; EKind Kind; diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 1f86dd874d7b..01a34d3d71f7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1561,11 +1561,10 @@ struct Schema : NIceDb::Schema { // NKikimrSchemeOp::TModifyScheme serialized as string struct PreparedCreationQuery : Column<14, NScheme::NTypeIds::String> {}; struct Permissions : Column<11, NScheme::NTypeIds::String> {}; - struct Metadata : Column<12, NScheme::NTypeIds::String> {}; - struct Changefeeds : Column<13, NScheme::NTypeIds::Json> {}; - struct Topics : Column<14, NScheme::NTypeIds::Json> {}; + struct Metadata : Column<12, NScheme::NTypeIds::String> {}; + struct Changefeeds : Column<15, NScheme::NTypeIds::String> { using Type = NKikimrSchemeOp::TImportTableChangefeeds; }; - struct State : Column<7, NScheme::NTypeIds::Byte> {}; + struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct NextIndexIdx : Column<9, NScheme::NTypeIds::Uint32> {}; struct Issue : Column<10, NScheme::NTypeIds::Utf8> {}; @@ -1582,6 +1581,7 @@ struct Schema : NIceDb::Schema { PreparedCreationQuery, Permissions, Metadata, + Changefeeds, State, WaitTxId, NextIndexIdx, diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index ce33e9ca21a7..546f4362153e 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -90,7 +90,7 @@ void FillChangefeedDescription(Ydb::Table::DescribeTableResult& out, bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, const Ydb::Table::Changefeed& in, Ydb::StatusIds::StatusCode& status, TString& error); bool FillChangefeedDescription(NKikimrSchemeOp::TCdcStreamDescription& out, - const Ydb::Table::ChangefeedDescription& in, Ydb::StatusIds::StatusCode& status, TString& error); + const Ydb::Table::ChangefeedDescription& in, Ydb::StatusIds::StatusCode& status, TString& error); // out void FillTableStats(Ydb::Table::DescribeTableResult& out, From 7b3c4dcb48ec06d7c7f9dd0c28d41d4735ca167b Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 31 Jan 2025 16:16:18 +0300 Subject: [PATCH 12/65] schemeshard schema --- .../flat_schemeshard.schema | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 8759c64389d1..8508d77bc393 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -6201,6 +6201,11 @@ "ColumnId": 14, "ColumnName": "PreparedCreationQuery", "ColumnType": "String" + }, + { + "ColumnId": 15, + "ColumnName": "Changefeeds", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -6220,7 +6225,8 @@ 11, 12, 13, - 14 + 14, + 15 ], "RoomID": 0, "Codec": 0, @@ -8124,4 +8130,4 @@ } } } -] \ No newline at end of file +] From 58659d2540848f2deb1323adda52dd26b72f38dd Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 3 Feb 2025 12:17:16 +0300 Subject: [PATCH 13/65] more reqs --- .../schemeshard_import__create.cpp | 18 ++++----- .../schemeshard_import_flow_proposals.cpp | 39 ++++++++----------- .../schemeshard_import_flow_proposals.h | 2 +- .../tx/schemeshard/schemeshard_info_types.h | 2 + .../tx/schemeshard/ut_restore/ut_restore.cpp | 35 ++++++++++++++++- 5 files changed, 63 insertions(+), 33 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 6088e15423f7..6f67879bbc59 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -371,28 +371,23 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(propose); Send(Self->SelfId(), std::move(propose)); - - CreateChangefeeds(importInfo, itemIdx, txId); } - void CreateChangefeeds(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { + void CreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); item.SubState = ESubState::Proposed; - LOG_I("TImport::TTxProgress: CreateChangefeeds propose" + LOG_I("TImport::TTxProgress: CreateChangefeed propose" << ": info# " << importInfo->ToString() << ", item# " << item.ToString(itemIdx) << ", txId# " << txId); Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); - auto proposes = CreateChangefeedsProposes(Self, txId, item); - - for (auto& propose : proposes) { - Send(Self->SelfId(), std::move(propose)); - } + auto propose = CreateChangefeedPropose(Self, txId, item); + Send(Self->SelfId(), std::move(propose)); } void ExecutePreparedQuery(TTransactionContext& txc, TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { @@ -1004,6 +999,11 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase BuildIndex(importInfo, i, txId); itemIdx = i; break; + + case EState::CreateChangefeed: + CreateChangefeed(importInfo, i, txId); + itemIdx = i; + break; default: break; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index c078a7bd0d58..c0adc55a9d18 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -86,28 +86,23 @@ THolder CreateTablePropose( return CreateTablePropose(ss, txId, importInfo, itemIdx, unused); } -TVector> CreateChangefeedsProposes( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item) { - const auto& changefeeds = item.Changefeeds; - TVector> result; - result.reserve(changefeeds.size()); - - for (const auto& [changefeed, topic]: changefeeds) { - auto propose = MakeHolder(ui64(txId), ss->TabletID()); - auto& record = propose->Record; - auto& modifyScheme = *record.AddTransaction(); - auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); - - TString error; - Ydb::StatusIds::StatusCode status; - - auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); - - Y_ABORT_UNLESS(FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)); - - cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); - result.push_back(std::move(propose)); - } - return result; +THolder CreateChangefeedPropose( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item) { + const auto& [changefeed, topic] = item.Changefeed; + auto propose = MakeHolder(ui64(txId), ss->TabletID()); + auto& record = propose->Record; + auto& modifyScheme = *record.AddTransaction(); + auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); + + TString error; + Ydb::StatusIds::StatusCode status; + + auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); + + Y_ABORT_UNLESS(FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)); + + cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); + + return propose; } static NKikimrSchemeOp::TTableDescription GetTableDescription(TSchemeShard* ss, const TPathId& pathId) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h index b2b50a6fa94e..9a77e0bb5fd3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h @@ -20,7 +20,7 @@ THolder CreateTablePropose( ui32 itemIdx ); -TVector> CreateChangefeedsProposes( +THolder CreateChangefeedPropose( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 6bc4f28052eb..0f2cb69d6335 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2828,6 +2828,7 @@ struct TImportInfo: public TSimpleRefCount { CreateSchemeObject, Transferring, BuildIndexes, + CreateChangefeed, Done = 240, Cancellation = 250, Cancelled = 251, @@ -2857,6 +2858,7 @@ struct TImportInfo: public TSimpleRefCount { TMaybeFail Permissions; NBackup::TMetadata Metadata; TVector Changefeeds; + int NextChangefeedIdx = 0; EState State = EState::GetScheme; ESubState SubState = ESubState::AllocateTxId; diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index f51381ef4197..4df091ee4080 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -145,6 +145,7 @@ namespace { TString Scheme; TString CreationQuery; TString Permissions; + TVector Changefeeds; TVector Data; TTestDataWithScheme() = default; @@ -251,12 +252,14 @@ namespace { const TTypedScheme& typedScheme, const TVector>& shardsConfig = {{"a", 1}}, const TString& permissions = "", - const TString& metadata = "" + const TString& metadata = "", + const TVector changefeeds = {} ) { TTestDataWithScheme result; result.Type = typedScheme.Type; result.Permissions = permissions; result.Metadata = metadata; + result.Changefeeds = changefeeds; switch (typedScheme.Type) { case EPathTypeTable: @@ -5061,4 +5064,34 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )" ); } + + Y_UNIT_TEST(ShouldSucceedOnSingleShardTableWithChangefeed) { + TTestBasicRuntime runtime; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}); + + Run(runtime, ConvertTestData(data), R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )"); + + auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); + NKqp::CompareYson(data.Data[0].YsonStr, content); + } } From 82c4acb87857db3eda2d84b383b9921a544ac7af Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 3 Feb 2025 13:20:46 +0300 Subject: [PATCH 14/65] requests one after another --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 1 + ydb/core/tx/schemeshard/schemeshard_import.cpp | 4 ++++ .../tx/schemeshard/schemeshard_import__create.cpp | 13 +++++++++++++ .../schemeshard_import_flow_proposals.cpp | 3 ++- ydb/core/tx/schemeshard/schemeshard_info_types.h | 4 ++-- ydb/core/tx/schemeshard/schemeshard_schema.h | 2 ++ ydb/public/api/protos/ydb_import.proto | 1 + 7 files changed, 25 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index a50c6e981c73..c2f688f5a4d7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4483,6 +4483,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase { item.State = static_cast(rowset.GetValue()); item.WaitTxId = rowset.GetValueOrDefault(InvalidTxId); item.NextIndexIdx = rowset.GetValueOrDefault(0); + item.NextChangefeedIdx = rowset.GetValueOrDefault(0); item.Issue = rowset.GetValueOrDefault(TString()); if (item.WaitTxId != InvalidTxId) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index d0fbc37cf47f..b68b097268f3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -73,6 +73,9 @@ void TSchemeShard::FromXxportInfo(NKikimrImport::TImport& import, const TImportI case TImportInfo::EState::BuildIndexes: import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES); break; + case TImportInfo::EState::CreateChangefeed: + import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_CREATE_CHANGEFEEDS); + break; case TImportInfo::EState::Done: import.SetProgress(Ydb::Import::ImportProgress::PROGRESS_DONE); break; @@ -163,6 +166,7 @@ void TSchemeShard::PersistImportItemState(NIceDb::TNiceDb& db, const TImportInfo NIceDb::TUpdate(static_cast(item.State)), NIceDb::TUpdate(item.WaitTxId), NIceDb::TUpdate(item.NextIndexIdx), + NIceDb::TUpdate(item.NextChangefeedIdx), NIceDb::TUpdate(item.Issue) ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 6f67879bbc59..038fb79bbf12 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1229,6 +1229,19 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } else { if (++item.NextIndexIdx < item.Scheme.indexes_size()) { AllocateTxId(importInfo, itemIdx); + } else { + item.State = EState::CreateChangefeed; + } + } + break; + + case EState::CreateChangefeed: + if (const auto issue = GetIssues(TIndexBuildId(ui64(txId)))) { + item.Issue = *issue; + Cancel(importInfo, itemIdx, "issues during changefeed creating"); + } else { + if (item.NextChangefeedIdx++ < item.Changefeeds.size()) { + AllocateTxId(importInfo, itemIdx); } else { item.State = EState::Done; } diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index c0adc55a9d18..8f79a4b0c659 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -87,7 +87,8 @@ THolder CreateTablePropose( } THolder CreateChangefeedPropose( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item) { - const auto& [changefeed, topic] = item.Changefeed; + Y_ABORT_UNLESS(item.NextChangefeedIdx < item.Changefeeds.size()); + const auto& [changefeed, topic] = item.Changefeeds[item.NextChangefeedIdx]; auto propose = MakeHolder(ui64(txId), ss->TabletID()); auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 0f2cb69d6335..6f329ac86e41 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2858,7 +2858,6 @@ struct TImportInfo: public TSimpleRefCount { TMaybeFail Permissions; NBackup::TMetadata Metadata; TVector Changefeeds; - int NextChangefeedIdx = 0; EState State = EState::GetScheme; ESubState SubState = ESubState::AllocateTxId; @@ -2866,6 +2865,7 @@ struct TImportInfo: public TSimpleRefCount { TActorId SchemeGetter; TActorId SchemeQueryExecutor; int NextIndexIdx = 0; + ui64 NextChangefeedIdx = 0; TString Issue; int ViewCreationRetries = 0; @@ -2886,7 +2886,7 @@ struct TImportInfo: public TSimpleRefCount { static bool IsDone(const TItem& item); }; - + ui64 Id; // TxId from the original TEvCreateImportRequest TString Uid; EKind Kind; diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 01a34d3d71f7..30f3662fa24a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1567,6 +1567,7 @@ struct Schema : NIceDb::Schema { struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct NextIndexIdx : Column<9, NScheme::NTypeIds::Uint32> {}; + struct NextChangefeedIdx : Column<16, NScheme::NTypeIds::Uint64> {}; struct Issue : Column<10, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; @@ -1585,6 +1586,7 @@ struct Schema : NIceDb::Schema { State, WaitTxId, NextIndexIdx, + NextChangefeedIdx, Issue >; }; diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index 2a575a1f438a..7c9ec1cfbc7d 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -19,6 +19,7 @@ message ImportProgress { PROGRESS_DONE = 4; PROGRESS_CANCELLATION = 5; PROGRESS_CANCELLED = 6; + PROGRESS_CREATE_CHANGEFEEDS = 7; } } From 1cb0dcab2e57b2c96f33f8183510dae15bad9d21 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 3 Feb 2025 17:01:01 +0300 Subject: [PATCH 15/65] fix after review --- ydb/core/protos/flat_scheme_op.proto | 8 ++++++-- ydb/core/tx/schemeshard/schemeshard__init.cpp | 15 ++++++--------- ydb/core/tx/schemeshard/schemeshard_import.cpp | 6 ++++-- .../schemeshard_import_scheme_getter.cpp | 4 ++-- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 7c66beb28159..d99855936852 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -2126,7 +2126,11 @@ message TBackupBackupCollection { optional string TargetDir = 3; // must be set on Rewrite } +message TImportChangefeedTopic { + optional Ydb.Table.ChangefeedDescription ChangefeedDescription = 1; + optional Ydb.Topic.DescribeTopicResult Topic = 2; +} + message TImportTableChangefeeds { - repeated Ydb.Table.ChangefeedDescription ChangefeedDescriptions = 1; - repeated Ydb.Topic.DescribeTopicResult Topics = 2; + repeated TImportChangefeedTopic Changefeeds = 1; } diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index c2f688f5a4d7..a4681d7df761 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4465,17 +4465,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase { } if (rowset.HaveValue()) { - const auto& importTableChangefeeds = rowset.GetValue(); - const int count = importTableChangefeeds.GetChangefeedDescriptions().size(); - Y_ASSERT(importTableChangefeeds.GetTopics().size() == count); - + const auto& importTableChangefeeds = rowset.GetValue().GetChangefeeds(); TVector changefeeds; - changefeeds.reserve(count); + changefeeds.reserve(importTableChangefeeds.size()); - for (int i = 0; i < count; ++i) { - const Ydb::Table::ChangefeedDescription& changefeedImport = importTableChangefeeds.GetChangefeedDescriptions()[i]; - const Ydb::Topic::DescribeTopicResult& topicImport = importTableChangefeeds.GetTopics()[i]; - changefeeds.emplace_back(changefeedImport, topicImport); + for (const auto& changefeedAndTopic : importTableChangefeeds) { + const Ydb::Table::ChangefeedDescription& changefeed = changefeedAndTopic.GetChangefeedDescription(); + const Ydb::Topic::DescribeTopicResult& topic = changefeedAndTopic.GetTopic(); + changefeeds.emplace_back(changefeed, topic); } item.Changefeeds = std::move(changefeeds); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index b68b097268f3..bb5b8c337d60 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -197,8 +197,10 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf NKikimrSchemeOp::TImportTableChangefeeds changefeeds; for (const auto& [changefeed, topic] : item.Changefeeds) { - *changefeeds.MutableChangefeedDescriptions()->Add() = changefeed; - *changefeeds.MutableTopics()->Add() = topic; + NKikimrSchemeOp::TImportChangefeedTopic changefeedTopic; + *changefeedTopic.MutableChangefeedDescription() = changefeed; + *changefeedTopic.MutableTopic() = topic; + *changefeeds.MutableChangefeeds()->Add() = changefeedTopic; } db.Table().Key(importInfo->Id, itemIdx).Update( diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index be9e28bba9c1..c2b3f8a26e89 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -469,7 +469,7 @@ class TSchemeGetter: public TActorBootstrapped { Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); } - void ListingChangefeeds() { + void ListChangefeeds() { CreateClient(); ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); } @@ -496,7 +496,7 @@ class TSchemeGetter: public TActorBootstrapped { } void DownloadChangefeeds() { - ListingChangefeeds(); + ListChangefeeds(); } void ResetRetries() { From 9a1595df4979269631f5802ca5738b6332fb3b6c Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 3 Feb 2025 18:02:17 +0300 Subject: [PATCH 16/65] fix build --- .../schemeshard_import__create.cpp | 22 +++++++ .../schemeshard_import_flow_proposals.cpp | 10 +++- .../tx/schemeshard/ut_restore/ut_restore.cpp | 58 +++++++++---------- 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 038fb79bbf12..d0acd39974ac 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -524,6 +524,28 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase return true; } + // bool CancelChangefeedCreating(TImportInfo::TPtr importInfo, ui32 itemIdx) { + // Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); + // const auto& item = importInfo->Items.at(itemIdx); + + // if (item.WaitTxId == InvalidTxId) { + // if (item.SubState == ESubState::Proposed) { + // importInfo->State = EState::Cancellation; + // } + + // return false; + // } + + // importInfo->State = EState::Cancellation; + + // LOG_I("TImport::TTxProgress: cancel index building" + // << ": info# " << importInfo->ToString() + // << ", item# " << item.ToString(itemIdx)); + + // Send(Self->SelfId(), CancelIndexBuildPropose(Self, importInfo, item.WaitTxId), 0, importInfo->Id); + // return true; + // } + void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 8f79a4b0c659..11679dc355fa 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -86,7 +86,11 @@ THolder CreateTablePropose( return CreateTablePropose(ss, txId, importInfo, itemIdx, unused); } -THolder CreateChangefeedPropose( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item) { +THolder CreateChangefeedPropose( + TSchemeShard* ss, + TTxId txId, + const TImportInfo::TItem& item +) { Y_ABORT_UNLESS(item.NextChangefeedIdx < item.Changefeeds.size()); const auto& [changefeed, topic] = item.Changefeeds[item.NextChangefeedIdx]; auto propose = MakeHolder(ui64(txId), ss->TabletID()); @@ -99,7 +103,9 @@ THolder CreateChangefeedPropose( TSc auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); - Y_ABORT_UNLESS(FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)); + if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { + return nullptr; + } cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 4df091ee4080..9bd1f5d80c8f 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5065,33 +5065,33 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { ); } - Y_UNIT_TEST(ShouldSucceedOnSingleShardTableWithChangefeed) { - TTestBasicRuntime runtime; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 1}}); - - Run(runtime, ConvertTestData(data), R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )"); - - auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); - NKqp::CompareYson(data.Data[0].YsonStr, content); - } + // Y_UNIT_TEST(ShouldSucceedOnSingleShardTableWithChangefeed) { + // TTestBasicRuntime runtime; + + // const auto data = GenerateTestData(R"( + // columns { + // name: "key" + // type { optional_type { item { type_id: UTF8 } } } + // } + // columns { + // name: "value" + // type { optional_type { item { type_id: UTF8 } } } + // } + // primary_key: "key" + // )", {{"a", 1}}); + + // Run(runtime, ConvertTestData(data), R"( + // ImportFromS3Settings { + // endpoint: "localhost:%d" + // scheme: HTTP + // items { + // source_prefix: "" + // destination_path: "/MyRoot/Table" + // } + // } + // )"); + + // auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); + // NKqp::CompareYson(data.Data[0].YsonStr, content); + // } } From f742d9236b77e00be9d99a70f7eb55128596ad19 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 09:47:37 +0300 Subject: [PATCH 17/65] new state in python test --- contrib/python/ydb/py2/ydb/import_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/python/ydb/py2/ydb/import_client.py b/contrib/python/ydb/py2/ydb/import_client.py index a11d77a0c688..368f1d30579f 100644 --- a/contrib/python/ydb/py2/ydb/import_client.py +++ b/contrib/python/ydb/py2/ydb/import_client.py @@ -27,6 +27,7 @@ class ImportProgress(enum.IntEnum): DONE = 4 CANCELLATION = 5 CANCELLED = 6 + CREATE_CHANGEFEEDS = 7 def _initialize_progresses(): From 6920f436cdc9fc7dd7e7b1e711d7828733265754 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 12:25:43 +0300 Subject: [PATCH 18/65] update schema --- .../flat_schemeshard.schema | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 8508d77bc393..e28dc3305e54 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -6206,6 +6206,11 @@ "ColumnId": 15, "ColumnName": "Changefeeds", "ColumnType": "String" + }, + { + "ColumnId": 16, + "ColumnName": "NextChangefeedIdx", + "ColumnType": "Uint64" } ], "ColumnsDropped": [], @@ -6226,7 +6231,8 @@ 12, 13, 14, - 15 + 15, + 16 ], "RoomID": 0, "Codec": 0, From 957ee2be5bf6111fe5f09ab1d9b162aaae2c16ec Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 12:40:40 +0300 Subject: [PATCH 19/65] feature flag added --- ydb/core/protos/feature_flags.proto | 1 + ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 8553c96c292e..e8e53e5f533d 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -188,4 +188,5 @@ message TFeatureFlags { optional bool EnableTopicTransfer = 163 [default = false]; optional bool EnableViewExport = 164 [default = false]; optional bool EnableColumnStore = 165 [default = false]; + optional bool EnableChangefeedsImport = 166 [default = false]; } diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index d0acd39974ac..6826dddd42b6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1252,7 +1252,10 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase if (++item.NextIndexIdx < item.Scheme.indexes_size()) { AllocateTxId(importInfo, itemIdx); } else { - item.State = EState::CreateChangefeed; + if (AppData()->FeatureFlags.GetEnableChangefeedsImport()) { + + } + item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; } } break; From dc96c314a323cc42723a89824123214a6f4e7c52 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 13:48:35 +0300 Subject: [PATCH 20/65] cli & feature flag --- ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 3 --- ydb/public/sdk/cpp/client/ydb_import/import.h | 1 + ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp | 4 ++-- ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h | 1 + ydb/public/sdk/cpp/src/client/proto/accessor.cpp | 2 ++ 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 6826dddd42b6..d259ae708e1c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1252,9 +1252,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase if (++item.NextIndexIdx < item.Scheme.indexes_size()) { AllocateTxId(importInfo, itemIdx); } else { - if (AppData()->FeatureFlags.GetEnableChangefeedsImport()) { - - } item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; } } diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.h b/ydb/public/sdk/cpp/client/ydb_import/import.h index 23247edf56c0..3ea0a52016cc 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.h +++ b/ydb/public/sdk/cpp/client/ydb_import/import.h @@ -17,6 +17,7 @@ enum class EImportProgress { Done = 4, Cancellation = 5, Cancelled = 6, + CreateChangefeed = 7, Unknown = Max(), }; diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp index 5f82e9236c14..e66ceb5d8a6e 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp @@ -122,8 +122,8 @@ NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress:: return NImport::EImportProgress::Preparing; case Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA: return NImport::EImportProgress::TransferData; - case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: - return NImport::EImportProgress::BuildIndexes; + case Ydb::Import::ImportProgress::PROGRESS_CREATE_CHANGEFEEDS: + return NImport::EImportProgress::CreateChangefeed; case Ydb::Import::ImportProgress::PROGRESS_DONE: return NImport::EImportProgress::Done; case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h index 3e6b191cbf2c..1a45ddbb05d5 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h @@ -17,6 +17,7 @@ enum class EImportProgress { Done = 4, Cancellation = 5, Cancelled = 6, + CreateChangefeed = 7, Unknown = std::numeric_limits::max(), }; diff --git a/ydb/public/sdk/cpp/src/client/proto/accessor.cpp b/ydb/public/sdk/cpp/src/client/proto/accessor.cpp index 9adbba5c0e6d..c34a33e8b939 100644 --- a/ydb/public/sdk/cpp/src/client/proto/accessor.cpp +++ b/ydb/public/sdk/cpp/src/client/proto/accessor.cpp @@ -124,6 +124,8 @@ NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress:: return NImport::EImportProgress::TransferData; case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: return NImport::EImportProgress::BuildIndexes; + case Ydb::Import::ImportProgress::PROGRESS_CREATE_CHANGEFEEDS: + return NImport::EImportProgress::CreateChangefeed; case Ydb::Import::ImportProgress::PROGRESS_DONE: return NImport::EImportProgress::Done; case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: From 2a447f5fe21e27447b2aab1f6e941c688a038ea2 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 14:11:48 +0300 Subject: [PATCH 21/65] removed cancel --- .../schemeshard_import__create.cpp | 33 ++----------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index d259ae708e1c..d23f4788562f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -524,28 +524,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase return true; } - // bool CancelChangefeedCreating(TImportInfo::TPtr importInfo, ui32 itemIdx) { - // Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); - // const auto& item = importInfo->Items.at(itemIdx); - - // if (item.WaitTxId == InvalidTxId) { - // if (item.SubState == ESubState::Proposed) { - // importInfo->State = EState::Cancellation; - // } - - // return false; - // } - - // importInfo->State = EState::Cancellation; - - // LOG_I("TImport::TTxProgress: cancel index building" - // << ": info# " << importInfo->ToString() - // << ", item# " << item.ToString(itemIdx)); - - // Send(Self->SelfId(), CancelIndexBuildPropose(Self, importInfo, item.WaitTxId), 0, importInfo->Id); - // return true; - // } - void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); @@ -1258,15 +1236,10 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase break; case EState::CreateChangefeed: - if (const auto issue = GetIssues(TIndexBuildId(ui64(txId)))) { - item.Issue = *issue; - Cancel(importInfo, itemIdx, "issues during changefeed creating"); + if (++item.NextChangefeedIdx < item.Changefeeds.size()) { + AllocateTxId(importInfo, itemIdx); } else { - if (item.NextChangefeedIdx++ < item.Changefeeds.size()) { - AllocateTxId(importInfo, itemIdx); - } else { - item.State = EState::Done; - } + item.State = EState::Done; } break; From c3d6c22c700661a37ac74ec6e18e258a7d9c23d4 Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Tue, 4 Feb 2025 14:39:15 +0300 Subject: [PATCH 22/65] Update ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp Co-authored-by: Ilnaz Nizametdinov --- ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 11679dc355fa..3e14251f49cf 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -107,7 +107,7 @@ THolder CreateChangefeedPropose( return nullptr; } - cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); + cdcStream.SetRetentionPeriodSeconds(topic.retention_period().seconds()); return propose; } From 8435331314adc082df2e6846f50169fe2ccdf911 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 14:44:29 +0300 Subject: [PATCH 23/65] ui32 & replace --- .../schemeshard_import_flow_proposals.cpp | 52 +++++++++---------- .../schemeshard_import_flow_proposals.h | 12 ++--- .../tx/schemeshard/schemeshard_info_types.h | 2 +- ydb/core/tx/schemeshard/schemeshard_schema.h | 2 +- .../flat_schemeshard.schema | 2 +- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 11679dc355fa..abd9611537a9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -86,32 +86,6 @@ THolder CreateTablePropose( return CreateTablePropose(ss, txId, importInfo, itemIdx, unused); } -THolder CreateChangefeedPropose( - TSchemeShard* ss, - TTxId txId, - const TImportInfo::TItem& item -) { - Y_ABORT_UNLESS(item.NextChangefeedIdx < item.Changefeeds.size()); - const auto& [changefeed, topic] = item.Changefeeds[item.NextChangefeedIdx]; - auto propose = MakeHolder(ui64(txId), ss->TabletID()); - auto& record = propose->Record; - auto& modifyScheme = *record.AddTransaction(); - auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); - - TString error; - Ydb::StatusIds::StatusCode status; - - auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); - - if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { - return nullptr; - } - - cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); - - return propose; -} - static NKikimrSchemeOp::TTableDescription GetTableDescription(TSchemeShard* ss, const TPathId& pathId) { auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId); auto record = desc->GetRecord(); @@ -257,5 +231,31 @@ THolder CancelIndexBuildPropose( return MakeHolder(ui64(indexBuildId), domainPath.PathString(), ui64(indexBuildId)); } +THolder CreateChangefeedPropose( + TSchemeShard* ss, + TTxId txId, + const TImportInfo::TItem& item +) { + Y_ABORT_UNLESS(static_cast(item.NextChangefeedIdx) < item.Changefeeds.size()); + const auto& [changefeed, topic] = item.Changefeeds[item.NextChangefeedIdx]; + auto propose = MakeHolder(ui64(txId), ss->TabletID()); + auto& record = propose->Record; + auto& modifyScheme = *record.AddTransaction(); + auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); + + TString error; + Ydb::StatusIds::StatusCode status; + + auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); + + if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { + return nullptr; + } + + cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); + + return propose; +} + } // NSchemeShard } // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h index 9a77e0bb5fd3..26ac6e2825cd 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.h @@ -20,12 +20,6 @@ THolder CreateTablePropose( ui32 itemIdx ); -THolder CreateChangefeedPropose( - TSchemeShard* ss, - TTxId txId, - const TImportInfo::TItem& item -); - THolder RestorePropose( TSchemeShard* ss, TTxId txId, @@ -52,5 +46,11 @@ THolder CancelIndexBuildPropose( TTxId indexBuildId ); +THolder CreateChangefeedPropose( + TSchemeShard* ss, + TTxId txId, + const TImportInfo::TItem& item +); + } // NSchemeShard } // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 468229fea65e..aaa9843cbe53 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2865,7 +2865,7 @@ struct TImportInfo: public TSimpleRefCount { TActorId SchemeGetter; TActorId SchemeQueryExecutor; int NextIndexIdx = 0; - ui64 NextChangefeedIdx = 0; + int NextChangefeedIdx = 0; TString Issue; int ViewCreationRetries = 0; diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 30f3662fa24a..c18ef8c78713 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1567,7 +1567,7 @@ struct Schema : NIceDb::Schema { struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct NextIndexIdx : Column<9, NScheme::NTypeIds::Uint32> {}; - struct NextChangefeedIdx : Column<16, NScheme::NTypeIds::Uint64> {}; + struct NextChangefeedIdx : Column<16, NScheme::NTypeIds::Uint32> {}; struct Issue : Column<10, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index e28dc3305e54..a7683c6ced7b 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -6210,7 +6210,7 @@ { "ColumnId": 16, "ColumnName": "NextChangefeedIdx", - "ColumnType": "Uint64" + "ColumnType": "Uint32" } ], "ColumnsDropped": [], From c6e41564d0510874010212c70e45723f830d88b1 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 4 Feb 2025 17:27:17 +0300 Subject: [PATCH 24/65] fix review --- ydb/core/protos/flat_scheme_op.proto | 9 +++-- ydb/core/tx/schemeshard/schemeshard__init.cpp | 2 +- .../tx/schemeshard/schemeshard_import.cpp | 2 +- .../schemeshard_import__create.cpp | 36 +++++++++---------- .../schemeshard_import_flow_proposals.cpp | 34 ++++++++++++++++-- .../tx/schemeshard/schemeshard_info_types.h | 2 +- 6 files changed, 56 insertions(+), 29 deletions(-) diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index d99855936852..5b6d3e55fbc9 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -2126,11 +2126,10 @@ message TBackupBackupCollection { optional string TargetDir = 3; // must be set on Rewrite } -message TImportChangefeedTopic { - optional Ydb.Table.ChangefeedDescription ChangefeedDescription = 1; - optional Ydb.Topic.DescribeTopicResult Topic = 2; -} - message TImportTableChangefeeds { + message TImportChangefeedTopic { + optional Ydb.Table.ChangefeedDescription Changefeed = 1; + optional Ydb.Topic.DescribeTopicResult Topic = 2; + } repeated TImportChangefeedTopic Changefeeds = 1; } diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index a4681d7df761..39a9925280f8 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4470,7 +4470,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase { changefeeds.reserve(importTableChangefeeds.size()); for (const auto& changefeedAndTopic : importTableChangefeeds) { - const Ydb::Table::ChangefeedDescription& changefeed = changefeedAndTopic.GetChangefeedDescription(); + const Ydb::Table::ChangefeedDescription& changefeed = changefeedAndTopic.GetChangefeed(); const Ydb::Topic::DescribeTopicResult& topic = changefeedAndTopic.GetTopic(); changefeeds.emplace_back(changefeed, topic); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index bb5b8c337d60..af059c804ec9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -198,7 +198,7 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf for (const auto& [changefeed, topic] : item.Changefeeds) { NKikimrSchemeOp::TImportChangefeedTopic changefeedTopic; - *changefeedTopic.MutableChangefeedDescription() = changefeed; + *changefeedTopic.MutableChangefeed() = changefeed; *changefeedTopic.MutableTopic() = topic; *changefeeds.MutableChangefeeds()->Add() = changefeedTopic; } diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index d23f4788562f..11e2519bd5d5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -373,23 +373,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Send(Self->SelfId(), std::move(propose)); } - void CreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { - Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); - auto& item = importInfo->Items.at(itemIdx); - - item.SubState = ESubState::Proposed; - - LOG_I("TImport::TTxProgress: CreateChangefeed propose" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx) - << ", txId# " << txId); - - Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); - - auto propose = CreateChangefeedPropose(Self, txId, item); - Send(Self->SelfId(), std::move(propose)); - } - void ExecutePreparedQuery(TTransactionContext& txc, TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items[itemIdx]; @@ -524,6 +507,23 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase return true; } + void CreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { + Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); + auto& item = importInfo->Items.at(itemIdx); + + item.SubState = ESubState::Proposed; + + LOG_I("TImport::TTxProgress: CreateChangefeed propose" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx) + << ", txId# " << txId); + + Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); + + auto propose = CreateChangefeedPropose(Self, txId, item); + Send(Self->SelfId(), std::move(propose)); + } + void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); @@ -1236,7 +1236,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase break; case EState::CreateChangefeed: - if (++item.NextChangefeedIdx < item.Changefeeds.size()) { + if (static_cast(++item.NextChangefeedIdx) < item.Changefeeds.size()) { AllocateTxId(importInfo, itemIdx); } else { item.State = EState::Done; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index abd9611537a9..d0314545f644 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -17,7 +17,7 @@ THolder CreateTablePropose( ) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); const auto& item = importInfo->Items.at(itemIdx); - + auto propose = MakeHolder(ui64(txId), ss->TabletID()); auto& record = propose->Record; @@ -242,6 +242,7 @@ THolder CreateChangefeedPropose( auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); + // cdcStream.SetTableName(changefeed.); TString error; Ydb::StatusIds::StatusCode status; @@ -251,8 +252,35 @@ THolder CreateChangefeedPropose( if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { return nullptr; } - - cdcStream.SetRetentionPeriodSeconds(topic.Getretention_period().seconds()); + + if (topic.has_retention_period()) { + cdcStream.SetRetentionPeriodSeconds(topic.retention_period().seconds()); + } + + if (topic.has_partitioning_settings()) { + i64 minActivePartitions = + topic.partitioning_settings().min_active_partitions(); + if (minActivePartitions < 0) { + return nullptr; + } else if (minActivePartitions == 0) { + minActivePartitions = 1; + } + cdcStream.SetTopicPartitions(minActivePartitions); + + if (topic.partitioning_settings().has_auto_partitioning_settings()) { + auto& partitioningSettings = topic.partitioning_settings().auto_partitioning_settings(); + cdcStream.SetTopicAutoPartitioning(partitioningSettings.strategy() != ::Ydb::Topic::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_DISABLED); + + i64 maxActivePartitions = + topic.partitioning_settings().max_active_partitions(); + if (maxActivePartitions < 0) { + return nullptr; + } else if (maxActivePartitions == 0) { + maxActivePartitions = 50; + } + cdcStream.SetMaxPartitionCount(maxActivePartitions); + } + } return propose; } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index aaa9843cbe53..9ec09ee51392 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2839,7 +2839,7 @@ struct TImportInfo: public TSimpleRefCount { }; struct TChangefeedImportDescriptions { - Ydb::Table::ChangefeedDescription ChangefeedDescription; + Ydb::Table::ChangefeedDescription Changefeed; Ydb::Topic::DescribeTopicResult Topic; }; From a54b586771f4f27d65048b0709dffa102e694e71 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 5 Feb 2025 10:21:01 +0300 Subject: [PATCH 25/65] SetTableName --- ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index d0314545f644..f1b16cea659a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -242,7 +242,7 @@ THolder CreateChangefeedPropose( auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); - // cdcStream.SetTableName(changefeed.); + cdcStream.SetTableName(item.DstPathName); //? TString error; Ydb::StatusIds::StatusCode status; From b671aa735cba0b21cd95e14f91ea481b6e98c886 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 5 Feb 2025 10:24:49 +0300 Subject: [PATCH 26/65] removed extra spaces --- ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index f1b16cea659a..c1361cc91fcd 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -17,7 +17,7 @@ THolder CreateTablePropose( ) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); const auto& item = importInfo->Items.at(itemIdx); - + auto propose = MakeHolder(ui64(txId), ss->TabletID()); auto& record = propose->Record; From cd27fff7191007ef108c101a2cbe29f3b144e25a Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 5 Feb 2025 10:48:30 +0300 Subject: [PATCH 27/65] removed convert to TVector --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 11 +---------- ydb/core/tx/schemeshard/schemeshard_import.cpp | 11 +---------- .../tx/schemeshard/schemeshard_import__create.cpp | 2 +- .../schemeshard/schemeshard_import_flow_proposals.cpp | 6 ++++-- .../schemeshard/schemeshard_import_scheme_getter.cpp | 4 ++-- ydb/core/tx/schemeshard/schemeshard_info_types.h | 2 +- 6 files changed, 10 insertions(+), 26 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 39a9925280f8..a96cb961654e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4465,16 +4465,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase { } if (rowset.HaveValue()) { - const auto& importTableChangefeeds = rowset.GetValue().GetChangefeeds(); - TVector changefeeds; - changefeeds.reserve(importTableChangefeeds.size()); - - for (const auto& changefeedAndTopic : importTableChangefeeds) { - const Ydb::Table::ChangefeedDescription& changefeed = changefeedAndTopic.GetChangefeed(); - const Ydb::Topic::DescribeTopicResult& topic = changefeedAndTopic.GetTopic(); - changefeeds.emplace_back(changefeed, topic); - } - item.Changefeeds = std::move(changefeeds); + item.Changefeeds = std::move(rowset.GetValue()); } item.State = static_cast(rowset.GetValue()); diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index 6dce72b0a618..05a002eee590 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -193,18 +193,9 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf db.Table().Key(importInfo->Id, itemIdx).Update( NIceDb::TUpdate(item.Metadata.Serialize()) ); - - NKikimrSchemeOp::TImportTableChangefeeds changefeeds; - - for (const auto& [changefeed, topic] : item.Changefeeds) { - NKikimrSchemeOp::TImportTableChangefeeds::TImportChangefeedTopic changefeedTopic; - *changefeedTopic.MutableChangefeed() = changefeed; - *changefeedTopic.MutableTopic() = topic; - *changefeeds.MutableChangefeeds()->Add() = changefeedTopic; - } db.Table().Key(importInfo->Id, itemIdx).Update( - NIceDb::TUpdate(changefeeds) + NIceDb::TUpdate(item.Changefeeds) ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 11e2519bd5d5..bcbf0986b260 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1236,7 +1236,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase break; case EState::CreateChangefeed: - if (static_cast(++item.NextChangefeedIdx) < item.Changefeeds.size()) { + if (++item.NextChangefeedIdx < item.Changefeeds.GetChangefeeds().size()) { AllocateTxId(importInfo, itemIdx); } else { item.State = EState::Done; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index c1361cc91fcd..16631460528a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -236,8 +236,10 @@ THolder CreateChangefeedPropose( TTxId txId, const TImportInfo::TItem& item ) { - Y_ABORT_UNLESS(static_cast(item.NextChangefeedIdx) < item.Changefeeds.size()); - const auto& [changefeed, topic] = item.Changefeeds[item.NextChangefeedIdx]; + Y_ABORT_UNLESS(item.NextChangefeedIdx < item.Changefeeds.GetChangefeeds().size()); + const auto& importChangefeedTopic = item.Changefeeds.GetChangefeeds()[item.NextChangefeedIdx]; + const auto& changefeed = importChangefeedTopic.GetChangefeed(); + const auto& topic = importChangefeedTopic.GetTopic(); auto propose = MakeHolder(ui64(txId), ss->TabletID()); auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 1454ef0bf7b5..4481092c6164 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -335,7 +335,7 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &changefeed)) { return Reply(false, "Cannot parse сhangefeed"); } - item.Changefeeds[IndexDownloadedChangefeed].Changefeed = std::move(changefeed); + *(*item.Changefeeds.MutableChangefeeds())[IndexDownloadedChangefeed].MutableChangefeed() = std::move(changefeed); auto nextStep = [this]() { HeadObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); @@ -371,7 +371,7 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &topic)) { return Reply(false, "Cannot parse topic"); } - item.Changefeeds[IndexDownloadedChangefeed].Topic = std::move(topic); + *(*item.Changefeeds.MutableChangefeeds())[IndexDownloadedChangefeed].MutableTopic() = std::move(topic); auto nextStep = [this]() { if (++IndexDownloadedChangefeed == ChangefeedsKeys.size()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 9ec09ee51392..753ec2ce2ba3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2857,7 +2857,7 @@ struct TImportInfo: public TSimpleRefCount { TMaybe PreparedCreationQuery; TMaybeFail Permissions; NBackup::TMetadata Metadata; - TVector Changefeeds; + NKikimrSchemeOp::TImportTableChangefeeds Changefeeds; EState State = EState::GetScheme; ESubState SubState = ESubState::AllocateTxId; From ad292e9c3a66381b354fba2a9e48182c97509361 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 5 Feb 2025 16:00:28 +0300 Subject: [PATCH 28/65] check tests after reply in ListObjects --- .../schemeshard_import_scheme_getter.cpp | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 4481092c6164..f79821dcf8a1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -396,13 +396,28 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleChangefeeds(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { + Y_UNUSED(ev); + // Reply(); const auto& result = ev.Get()->Get()->Result; - + // Cerr << "result1: " << result << Endl; + Cerr << "re1: " << result.GetError().GetMessage()<< Endl; + // Cerr << "re1: " << result.GetError().GetErrorType()<< Endl; + // Cerr << "re1: " << result.GetError().GetResponseCode()<< Endl; + Cerr << "re1: " << result.GetError().GetExceptionName()<< Endl; + Cerr << result.IsSuccess() << Endl; + // Cerr << result.GetError() << Endl; + // Cerr << "ev1: " << ev << Endl; + // Cerr << "ev2: " << ev.Get() << Endl; + // Cerr << "ev3: " << ev.Get()->Get() << Endl; + if (!result.IsSuccess()) { + Reply(); + return; + } LOG_D("HandleChangefeeds TEvExternalStorage::TEvListObjectResponse" << ": self# " << SelfId() << ", result# " << result); - if (!CheckResult(result, "ListObject")) { + if (!CheckResult(result, "ListObjects")) { return; } @@ -471,7 +486,10 @@ class TSchemeGetter: public TActorBootstrapped { void ListChangefeeds() { CreateClient(); - ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); + Cerr << "Key1: " << ImportInfo->Settings.items(ItemIdx).destination_path() << Endl; + Cerr << "Key1: " << ImportInfo->Settings.items(ItemIdx).source_prefix() << Endl; + Cerr << "bucj: " << ImportInfo->Settings.bucket() << Endl; + ListObjects("/"); } void Download(const TString& key) { From 69fe837b19152031be3a28a7806d1c7a7a1bac0b Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 6 Feb 2025 16:10:43 +0300 Subject: [PATCH 29/65] removed extra --- .../schemeshard_import_scheme_getter.cpp | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index f79821dcf8a1..72f2d30771b6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -396,23 +396,7 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleChangefeeds(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { - Y_UNUSED(ev); - // Reply(); const auto& result = ev.Get()->Get()->Result; - // Cerr << "result1: " << result << Endl; - Cerr << "re1: " << result.GetError().GetMessage()<< Endl; - // Cerr << "re1: " << result.GetError().GetErrorType()<< Endl; - // Cerr << "re1: " << result.GetError().GetResponseCode()<< Endl; - Cerr << "re1: " << result.GetError().GetExceptionName()<< Endl; - Cerr << result.IsSuccess() << Endl; - // Cerr << result.GetError() << Endl; - // Cerr << "ev1: " << ev << Endl; - // Cerr << "ev2: " << ev.Get() << Endl; - // Cerr << "ev3: " << ev.Get()->Get() << Endl; - if (!result.IsSuccess()) { - Reply(); - return; - } LOG_D("HandleChangefeeds TEvExternalStorage::TEvListObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -486,10 +470,7 @@ class TSchemeGetter: public TActorBootstrapped { void ListChangefeeds() { CreateClient(); - Cerr << "Key1: " << ImportInfo->Settings.items(ItemIdx).destination_path() << Endl; - Cerr << "Key1: " << ImportInfo->Settings.items(ItemIdx).source_prefix() << Endl; - Cerr << "bucj: " << ImportInfo->Settings.bucket() << Endl; - ListObjects("/"); + ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); } void Download(const TString& key) { From 872ae51f1de6d2a7924987cdbe5a8b1206d63586 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 7 Feb 2025 13:15:16 +0300 Subject: [PATCH 30/65] fix --- .../schemeshard_import_flow_proposals.cpp | 3 ++- .../schemeshard_import_scheme_getter.cpp | 20 ++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 16631460528a..d5e2456f87cd 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -244,7 +244,8 @@ THolder CreateChangefeedPropose( auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); - cdcStream.SetTableName(item.DstPathName); //? + Cerr << "TableName: " << item.DstPathName << Endl; + cdcStream.SetTableName("movies_chfd"); //? TString error; Ydb::StatusIds::StatusCode status; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 72f2d30771b6..e30a292d296e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -60,6 +60,7 @@ class TSchemeGetter: public TActorBootstrapped { } void HeadObject(const TString& key) { + Cerr << "HeadObject: " << key << Endl; auto request = Model::HeadObjectRequest() .WithKey(key); @@ -137,6 +138,7 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleChangefeed(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + Cerr << "Head HandleChangefeed " << IndexDownloadedChangefeed << Endl; const auto& result = ev->Get()->Result; LOG_D("HandleChangefeed TEvExternalStorage::TEvHeadObjectResponse" @@ -148,10 +150,12 @@ class TSchemeGetter: public TActorBootstrapped { } const auto contentLength = result.GetResult().GetContentLength(); + Y_ABORT_UNLESS(IndexDownloadedChangefeed < ChangefeedsKeys.size()); GetObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), std::make_pair(0, contentLength - 1)); } void HandleTopic(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + Cerr << "Head HandleTopic " << IndexDownloadedChangefeed << Endl; const auto& result = ev->Get()->Result; LOG_D("HandleTopic TEvExternalStorage::TEvHeadObjectResponse" @@ -163,10 +167,12 @@ class TSchemeGetter: public TActorBootstrapped { } const auto contentLength = result.GetResult().GetContentLength(); + Y_ABORT_UNLESS(IndexDownloadedChangefeed < ChangefeedsKeys.size()); GetObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed]), std::make_pair(0, contentLength - 1)); } void GetObject(const TString& key, const std::pair& range) { + Cerr << "GetObject: " << key << Endl; auto request = Model::GetObjectRequest() .WithKey(key) .WithRange(TStringBuilder() << "bytes=" << range.first << "-" << range.second); @@ -313,6 +319,7 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleChangefeed(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + Cerr << "Get HandleChangefeed " << IndexDownloadedChangefeed << Endl; const auto& msg = *ev->Get(); const auto& result = msg.Result; @@ -335,9 +342,11 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &changefeed)) { return Reply(false, "Cannot parse сhangefeed"); } - *(*item.Changefeeds.MutableChangefeeds())[IndexDownloadedChangefeed].MutableChangefeed() = std::move(changefeed); + Cerr << "An attempt to save changefeed (description)" << IndexDownloadedChangefeed + 1 << "/" << item.Changefeeds.ChangefeedsSize() << Endl; + *item.Changefeeds.MutableChangefeeds()->Add()->MutableChangefeed() = std::move(changefeed); auto nextStep = [this]() { + Become(&TThis::StateDownloadTopics); HeadObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); }; @@ -349,6 +358,7 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleTopic(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + Cerr << "Get HandleTopic " << IndexDownloadedChangefeed << Endl; const auto& msg = *ev->Get(); const auto& result = msg.Result; @@ -371,12 +381,14 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &topic)) { return Reply(false, "Cannot parse topic"); } - *(*item.Changefeeds.MutableChangefeeds())[IndexDownloadedChangefeed].MutableTopic() = std::move(topic); + Cerr << "An attempt to save changefeed (topic)" << IndexDownloadedChangefeed + 1 << "/" << item.Changefeeds.ChangefeedsSize(); + *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableTopic() = std::move(topic); auto nextStep = [this]() { - if (++IndexDownloadedChangefeed == ChangefeedsKeys.size()) { + if (++IndexDownloadedChangefeed >= ChangefeedsKeys.size()) { Reply(); } else { + Become(&TThis::StateDownloadChangefeeds); HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); } }; @@ -416,6 +428,7 @@ class TSchemeGetter: public TActorBootstrapped { } if (!ChangefeedsKeys.empty()) { + Y_ABORT_UNLESS(IndexDownloadedChangefeed < ChangefeedsKeys.size()); HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); } else { Reply(); @@ -470,6 +483,7 @@ class TSchemeGetter: public TActorBootstrapped { void ListChangefeeds() { CreateClient(); + Cerr << "ListChangefeeds" << Endl; ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); } From af768cd9e29bb3cfb6e6818533ede8d8d2bd7dcf Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 7 Feb 2025 19:08:03 +0300 Subject: [PATCH 31/65] working version --- .../schemeshard_import__create.cpp | 22 ++++++++++++++----- .../schemeshard_import_flow_proposals.cpp | 19 ++++++++++++---- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index bcbf0986b260..d0cc0ff17471 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -323,12 +323,14 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } else if (SchemeQueryResult) { OnSchemeQueryPreparation(txc); } else if (AllocateResult) { + Cerr << "DoExecute:OnAllocateResult&&" << Endl; OnAllocateResult(txc, ctx); } else if (ModifyResult) { OnModifyResult(txc, ctx); } else if (CreateIndexResult) { OnCreateIndexResult(txc, ctx); } else if (CompletedTxId) { + Cerr << "DoExecute:OnNotifyResult&&" << Endl; OnNotifyResult(txc, ctx); } else { Resume(txc, ctx); @@ -510,7 +512,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase void CreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); - + Cerr << "CreateChangefeed13232" << Endl; item.SubState = ESubState::Proposed; LOG_I("TImport::TTxProgress: CreateChangefeed propose" @@ -521,6 +523,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); auto propose = CreateChangefeedPropose(Self, txId, item); + if (!propose) Cerr << "NO CHNGF PROPOSE" << Endl; Send(Self->SelfId(), std::move(propose)); } @@ -972,6 +975,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (item.State) { case EState::CreateSchemeObject: + Cerr << "CreateSchemeObject!!!" << Endl; if (item.PreparedCreationQuery) { ExecutePreparedQuery(txc, importInfo, i, txId); itemIdx = i; @@ -1001,6 +1005,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase break; case EState::CreateChangefeed: + Cerr << "OnAllocateResult:CreateChangefeed" << Endl; CreateChangefeed(importInfo, i, txId); itemIdx = i; break; @@ -1200,29 +1205,32 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (item.State) { case EState::CreateSchemeObject: + Cerr << "CreateSchemeObject11" << Endl; if (IsCreatedByQuery(item)) { - item.State = EState::Done; + item.State = EState::CreateChangefeed; break; } - item.State = EState::Transferring; + item.State = EState::CreateChangefeed; AllocateTxId(importInfo, itemIdx); break; case EState::Transferring: + Cerr << "Transfering232" << Endl; if (const auto issue = GetIssues(item.DstPathId, txId)) { item.Issue = *issue; Cancel(importInfo, itemIdx, "issues during restore"); } else { if (item.NextIndexIdx < item.Scheme.indexes_size()) { - item.State = EState::BuildIndexes; + item.State = EState::CreateChangefeed; AllocateTxId(importInfo, itemIdx); } else { - item.State = EState::Done; + item.State = EState::CreateChangefeed; } } break; case EState::BuildIndexes: + Cerr << "BuildIndexes123232" << Endl; if (const auto issue = GetIssues(TIndexBuildId(ui64(txId)))) { item.Issue = *issue; Cancel(importInfo, itemIdx, "issues during index building"); @@ -1230,12 +1238,14 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase if (++item.NextIndexIdx < item.Scheme.indexes_size()) { AllocateTxId(importInfo, itemIdx); } else { - item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; + // item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed; + item.State = EState::CreateChangefeed; } } break; case EState::CreateChangefeed: + Cerr << "CreateChangefeed12" << Endl << "NextChangefeedIdx: " << item.NextChangefeedIdx; if (++item.NextChangefeedIdx < item.Changefeeds.GetChangefeeds().size()) { AllocateTxId(importInfo, itemIdx); } else { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index d5e2456f87cd..3ef3df6c85bc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -7,7 +7,6 @@ namespace NKikimr { namespace NSchemeShard { - THolder CreateTablePropose( TSchemeShard* ss, TTxId txId, @@ -231,6 +230,14 @@ THolder CancelIndexBuildPropose( return MakeHolder(ui64(indexBuildId), domainPath.PathString(), ui64(indexBuildId)); } +std::pair SplitPathIntoWorkingDirAndName(const TString& path) { + auto splitPos = path.find_last_of('/'); + if (splitPos == path.npos || splitPos + 1 == path.size()) { + ythrow yexception() << "wrong path format '" << path << "'" ; + } + return {path.substr(0, splitPos), path.substr(splitPos + 1)}; +} + THolder CreateChangefeedPropose( TSchemeShard* ss, TTxId txId, @@ -243,15 +250,18 @@ THolder CreateChangefeedPropose( auto propose = MakeHolder(ui64(txId), ss->TabletID()); auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); + modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream); auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); Cerr << "TableName: " << item.DstPathName << Endl; - cdcStream.SetTableName("movies_chfd"); //? + auto pair = SplitPathIntoWorkingDirAndName(item.DstPathName); + modifyScheme.SetWorkingDir(pair.first); + cdcStream.SetTableName(pair.second); TString error; Ydb::StatusIds::StatusCode status; auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); - + Cerr << "FillChangefeedDescription" << Endl; if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { return nullptr; } @@ -284,7 +294,8 @@ THolder CreateChangefeedPropose( cdcStream.SetMaxPartitionCount(maxActivePartitions); } } - + Cerr << "Create Changefeed proppse" << Endl; + Cerr << record << Endl; return propose; } From c1bf58eb6445ea681e9816713dc615b5b6ce34ab Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 7 Feb 2025 19:51:09 +0300 Subject: [PATCH 32/65] feature flag --- .../schemeshard_import__create.cpp | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index d0cc0ff17471..3dfbadd16486 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1169,6 +1169,9 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase void OnNotifyResult(TTransactionContext& txc, const TActorContext& ctx) { Y_ABORT_UNLESS(CompletedTxId); + + AppData()->FeatureFlags.SetEnableChangefeedsImport(true); + LOG_D("TImport::TTxProgress: OnNotifyResult" << ": txId# " << CompletedTxId); @@ -1205,32 +1208,32 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (item.State) { case EState::CreateSchemeObject: - Cerr << "CreateSchemeObject11" << Endl; if (IsCreatedByQuery(item)) { - item.State = EState::CreateChangefeed; + item.State = EState::Done; break; } - item.State = EState::CreateChangefeed; + item.State = EState::Transferring; AllocateTxId(importInfo, itemIdx); break; case EState::Transferring: - Cerr << "Transfering232" << Endl; if (const auto issue = GetIssues(item.DstPathId, txId)) { item.Issue = *issue; Cancel(importInfo, itemIdx, "issues during restore"); } else { if (item.NextIndexIdx < item.Scheme.indexes_size()) { - item.State = EState::CreateChangefeed; + item.State = EState::BuildIndexes; + AllocateTxId(importInfo, itemIdx); + } else if (item.NextChangefeedIdx < item.Changefeeds.changefeeds_size()) { + item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; AllocateTxId(importInfo, itemIdx); } else { - item.State = EState::CreateChangefeed; + item.State = EState::Done; } } break; case EState::BuildIndexes: - Cerr << "BuildIndexes123232" << Endl; if (const auto issue = GetIssues(TIndexBuildId(ui64(txId)))) { item.Issue = *issue; Cancel(importInfo, itemIdx, "issues during index building"); @@ -1238,14 +1241,12 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase if (++item.NextIndexIdx < item.Scheme.indexes_size()) { AllocateTxId(importInfo, itemIdx); } else { - // item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed; - item.State = EState::CreateChangefeed; + item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; } } break; case EState::CreateChangefeed: - Cerr << "CreateChangefeed12" << Endl << "NextChangefeedIdx: " << item.NextChangefeedIdx; if (++item.NextChangefeedIdx < item.Changefeeds.GetChangefeeds().size()) { AllocateTxId(importInfo, itemIdx); } else { From 7feb38d3d23b2b352aa4ccdfa59b133e57d516f4 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 7 Feb 2025 20:15:00 +0300 Subject: [PATCH 33/65] set table name --- .../schemeshard_import_flow_proposals.cpp | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 3ef3df6c85bc..6d4be6493a4e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -230,32 +230,26 @@ THolder CancelIndexBuildPropose( return MakeHolder(ui64(indexBuildId), domainPath.PathString(), ui64(indexBuildId)); } -std::pair SplitPathIntoWorkingDirAndName(const TString& path) { - auto splitPos = path.find_last_of('/'); - if (splitPos == path.npos || splitPos + 1 == path.size()) { - ythrow yexception() << "wrong path format '" << path << "'" ; - } - return {path.substr(0, splitPos), path.substr(splitPos + 1)}; -} - THolder CreateChangefeedPropose( TSchemeShard* ss, TTxId txId, const TImportInfo::TItem& item ) { Y_ABORT_UNLESS(item.NextChangefeedIdx < item.Changefeeds.GetChangefeeds().size()); + const auto& importChangefeedTopic = item.Changefeeds.GetChangefeeds()[item.NextChangefeedIdx]; const auto& changefeed = importChangefeedTopic.GetChangefeed(); const auto& topic = importChangefeedTopic.GetTopic(); + auto propose = MakeHolder(ui64(txId), ss->TabletID()); auto& record = propose->Record; auto& modifyScheme = *record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpCreateCdcStream); auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); - Cerr << "TableName: " << item.DstPathName << Endl; - auto pair = SplitPathIntoWorkingDirAndName(item.DstPathName); - modifyScheme.SetWorkingDir(pair.first); - cdcStream.SetTableName(pair.second); + + const TPath dstPath = TPath::Init(item.DstPathId, ss); + modifyScheme.SetWorkingDir( dstPath.Parent().PathString()); + cdcStream.SetTableName(dstPath.LeafName()); TString error; Ydb::StatusIds::StatusCode status; @@ -294,8 +288,6 @@ THolder CreateChangefeedPropose( cdcStream.SetMaxPartitionCount(maxActivePartitions); } } - Cerr << "Create Changefeed proppse" << Endl; - Cerr << record << Endl; return propose; } From cab1c258c2009ccc0e9086cec59209a8843da531 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 7 Feb 2025 20:29:29 +0300 Subject: [PATCH 34/65] clean up --- ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 6 ------ .../tx/schemeshard/schemeshard_import_flow_proposals.cpp | 1 - .../tx/schemeshard/schemeshard_import_scheme_getter.cpp | 9 --------- ydb/core/tx/schemeshard/schemeshard_info_types.h | 5 ----- ydb/core/tx/schemeshard/schemeshard_schema.h | 2 +- .../flat_schemeshard.schema | 2 +- 6 files changed, 2 insertions(+), 23 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 3dfbadd16486..01e744a67356 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -323,14 +323,12 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } else if (SchemeQueryResult) { OnSchemeQueryPreparation(txc); } else if (AllocateResult) { - Cerr << "DoExecute:OnAllocateResult&&" << Endl; OnAllocateResult(txc, ctx); } else if (ModifyResult) { OnModifyResult(txc, ctx); } else if (CreateIndexResult) { OnCreateIndexResult(txc, ctx); } else if (CompletedTxId) { - Cerr << "DoExecute:OnNotifyResult&&" << Endl; OnNotifyResult(txc, ctx); } else { Resume(txc, ctx); @@ -512,7 +510,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase void CreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx, TTxId txId) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); - Cerr << "CreateChangefeed13232" << Endl; item.SubState = ESubState::Proposed; LOG_I("TImport::TTxProgress: CreateChangefeed propose" @@ -523,7 +520,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); auto propose = CreateChangefeedPropose(Self, txId, item); - if (!propose) Cerr << "NO CHNGF PROPOSE" << Endl; Send(Self->SelfId(), std::move(propose)); } @@ -975,7 +971,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (item.State) { case EState::CreateSchemeObject: - Cerr << "CreateSchemeObject!!!" << Endl; if (item.PreparedCreationQuery) { ExecutePreparedQuery(txc, importInfo, i, txId); itemIdx = i; @@ -1005,7 +1000,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase break; case EState::CreateChangefeed: - Cerr << "OnAllocateResult:CreateChangefeed" << Endl; CreateChangefeed(importInfo, i, txId); itemIdx = i; break; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 6d4be6493a4e..a8c99b29a0e4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -255,7 +255,6 @@ THolder CreateChangefeedPropose( Ydb::StatusIds::StatusCode status; auto& cdcStreamDescription = *cdcStream.MutableStreamDescription(); - Cerr << "FillChangefeedDescription" << Endl; if (!FillChangefeedDescription(cdcStreamDescription, changefeed, status, error)) { return nullptr; } diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index e30a292d296e..4c0f09307525 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -60,7 +60,6 @@ class TSchemeGetter: public TActorBootstrapped { } void HeadObject(const TString& key) { - Cerr << "HeadObject: " << key << Endl; auto request = Model::HeadObjectRequest() .WithKey(key); @@ -138,7 +137,6 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleChangefeed(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { - Cerr << "Head HandleChangefeed " << IndexDownloadedChangefeed << Endl; const auto& result = ev->Get()->Result; LOG_D("HandleChangefeed TEvExternalStorage::TEvHeadObjectResponse" @@ -155,7 +153,6 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleTopic(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { - Cerr << "Head HandleTopic " << IndexDownloadedChangefeed << Endl; const auto& result = ev->Get()->Result; LOG_D("HandleTopic TEvExternalStorage::TEvHeadObjectResponse" @@ -172,7 +169,6 @@ class TSchemeGetter: public TActorBootstrapped { } void GetObject(const TString& key, const std::pair& range) { - Cerr << "GetObject: " << key << Endl; auto request = Model::GetObjectRequest() .WithKey(key) .WithRange(TStringBuilder() << "bytes=" << range.first << "-" << range.second); @@ -319,7 +315,6 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleChangefeed(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { - Cerr << "Get HandleChangefeed " << IndexDownloadedChangefeed << Endl; const auto& msg = *ev->Get(); const auto& result = msg.Result; @@ -342,7 +337,6 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &changefeed)) { return Reply(false, "Cannot parse сhangefeed"); } - Cerr << "An attempt to save changefeed (description)" << IndexDownloadedChangefeed + 1 << "/" << item.Changefeeds.ChangefeedsSize() << Endl; *item.Changefeeds.MutableChangefeeds()->Add()->MutableChangefeed() = std::move(changefeed); auto nextStep = [this]() { @@ -358,7 +352,6 @@ class TSchemeGetter: public TActorBootstrapped { } void HandleTopic(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { - Cerr << "Get HandleTopic " << IndexDownloadedChangefeed << Endl; const auto& msg = *ev->Get(); const auto& result = msg.Result; @@ -381,7 +374,6 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &topic)) { return Reply(false, "Cannot parse topic"); } - Cerr << "An attempt to save changefeed (topic)" << IndexDownloadedChangefeed + 1 << "/" << item.Changefeeds.ChangefeedsSize(); *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableTopic() = std::move(topic); auto nextStep = [this]() { @@ -483,7 +475,6 @@ class TSchemeGetter: public TActorBootstrapped { void ListChangefeeds() { CreateClient(); - Cerr << "ListChangefeeds" << Endl; ListObjects(ImportInfo->Settings.items(ItemIdx).source_prefix()); } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 753ec2ce2ba3..af78ff29b8a2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2838,11 +2838,6 @@ struct TImportInfo: public TSimpleRefCount { S3 = 0, }; - struct TChangefeedImportDescriptions { - Ydb::Table::ChangefeedDescription Changefeed; - Ydb::Topic::DescribeTopicResult Topic; - }; - struct TItem { enum class ESubState: ui8 { AllocateTxId = 0, diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index c18ef8c78713..86d035f21505 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1564,7 +1564,7 @@ struct Schema : NIceDb::Schema { struct Metadata : Column<12, NScheme::NTypeIds::String> {}; struct Changefeeds : Column<15, NScheme::NTypeIds::String> { using Type = NKikimrSchemeOp::TImportTableChangefeeds; }; - struct State : Column<7, NScheme::NTypeIds::Byte> {}; + struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct NextIndexIdx : Column<9, NScheme::NTypeIds::Uint32> {}; struct NextChangefeedIdx : Column<16, NScheme::NTypeIds::Uint32> {}; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index a7683c6ced7b..8cdfa535f662 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -8136,4 +8136,4 @@ } } } -] +] \ No newline at end of file From 07b78346c98df4273e217f555404d78063c15d4d Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 7 Feb 2025 20:38:52 +0300 Subject: [PATCH 35/65] clean up --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 2 +- ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 3 +-- ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index a96cb961654e..aaff7cd5bbc4 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4465,7 +4465,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase { } if (rowset.HaveValue()) { - item.Changefeeds = std::move(rowset.GetValue()); + item.Changefeeds = rowset.GetValue(); } item.State = static_cast(rowset.GetValue()); diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 01e744a67356..62a3b8fcf354 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -519,8 +519,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); - auto propose = CreateChangefeedPropose(Self, txId, item); - Send(Self->SelfId(), std::move(propose)); + Send(Self->SelfId(), CreateChangefeedPropose(Self, txId, item)); } void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index a8c99b29a0e4..45375414a37f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -7,6 +7,7 @@ namespace NKikimr { namespace NSchemeShard { + THolder CreateTablePropose( TSchemeShard* ss, TTxId txId, From 9f61dde59684e758c1fdd26fd666915335c6da18 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Feb 2025 12:06:11 +0300 Subject: [PATCH 36/65] fix review --- contrib/python/ydb/py2/ydb/import_client.py | 1 - .../tx/schemeshard/schemeshard_import__create.cpp | 11 ++++++++--- .../schemeshard/schemeshard_import_flow_proposals.cpp | 4 ++-- .../schemeshard/schemeshard_import_scheme_getter.cpp | 11 +++++++++-- ydb/public/sdk/cpp/client/ydb_import/import.h | 1 - ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp | 2 -- .../cpp/include/ydb-cpp-sdk/client/import/import.h | 2 +- 7 files changed, 20 insertions(+), 12 deletions(-) diff --git a/contrib/python/ydb/py2/ydb/import_client.py b/contrib/python/ydb/py2/ydb/import_client.py index 368f1d30579f..a11d77a0c688 100644 --- a/contrib/python/ydb/py2/ydb/import_client.py +++ b/contrib/python/ydb/py2/ydb/import_client.py @@ -27,7 +27,6 @@ class ImportProgress(enum.IntEnum): DONE = 4 CANCELLATION = 5 CANCELLED = 6 - CREATE_CHANGEFEEDS = 7 def _initialize_progresses(): diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 62a3b8fcf354..1f36726005f7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1217,8 +1217,9 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase if (item.NextIndexIdx < item.Scheme.indexes_size()) { item.State = EState::BuildIndexes; AllocateTxId(importInfo, itemIdx); - } else if (item.NextChangefeedIdx < item.Changefeeds.changefeeds_size()) { - item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; + } else if (item.NextChangefeedIdx < item.Changefeeds.changefeeds_size() && + AppData()->FeatureFlags.GetEnableChangefeedsImport()) { + item.State = EState::CreateChangefeed; AllocateTxId(importInfo, itemIdx); } else { item.State = EState::Done; @@ -1233,8 +1234,12 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } else { if (++item.NextIndexIdx < item.Scheme.indexes_size()) { AllocateTxId(importInfo, itemIdx); + } else if (item.NextChangefeedIdx < item.Changefeeds.changefeeds_size() && + AppData()->FeatureFlags.GetEnableChangefeedsImport()) { + item.State = EState::CreateChangefeed; + AllocateTxId(importInfo, itemIdx); } else { - item.State = AppData()->FeatureFlags.GetEnableChangefeedsImport() ? EState::CreateChangefeed : EState::Done; + item.State = EState::Done; } } break; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index 45375414a37f..30b63428d47f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -7,7 +7,7 @@ namespace NKikimr { namespace NSchemeShard { - + THolder CreateTablePropose( TSchemeShard* ss, TTxId txId, @@ -249,7 +249,7 @@ THolder CreateChangefeedPropose( auto& cdcStream = *modifyScheme.MutableCreateCdcStream(); const TPath dstPath = TPath::Init(item.DstPathId, ss); - modifyScheme.SetWorkingDir( dstPath.Parent().PathString()); + modifyScheme.SetWorkingDir(dstPath.Parent().PathString()); cdcStream.SetTableName(dstPath.LeafName()); TString error; diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 4c0f09307525..67a1353ca2d5 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -337,7 +337,13 @@ class TSchemeGetter: public TActorBootstrapped { if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &changefeed)) { return Reply(false, "Cannot parse сhangefeed"); } - *item.Changefeeds.MutableChangefeeds()->Add()->MutableChangefeed() = std::move(changefeed); + + if (IndexDownloadedChangefeed == item.Changefeeds.ChangefeedsSize()) { + *item.Changefeeds.MutableChangefeeds()->Add()->MutableChangefeed() = std::move(changefeed); + } else { + *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableChangefeed() = std::move(changefeed); + } + auto nextStep = [this]() { Become(&TThis::StateDownloadTopics); @@ -410,6 +416,7 @@ class TSchemeGetter: public TActorBootstrapped { } const auto& objects = result.GetResult().GetContents(); + ChangefeedsKeys.clear(); ChangefeedsKeys.reserve(objects.size()); for (const auto& obj : objects) { @@ -500,6 +507,7 @@ class TSchemeGetter: public TActorBootstrapped { } void DownloadChangefeeds() { + Become(&TThis::StateDownloadChangefeeds); ListChangefeeds(); } @@ -522,7 +530,6 @@ class TSchemeGetter: public TActorBootstrapped { void StartDownloadingChangefeeds() { ResetRetries(); DownloadChangefeeds(); - Become(&TThis::StateDownloadChangefeeds); } void StartValidatingChecksum(const TString& key, const TString& object, std::function checksumValidatedCallback) { diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.h b/ydb/public/sdk/cpp/client/ydb_import/import.h index 3ea0a52016cc..23247edf56c0 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.h +++ b/ydb/public/sdk/cpp/client/ydb_import/import.h @@ -17,7 +17,6 @@ enum class EImportProgress { Done = 4, Cancellation = 5, Cancelled = 6, - CreateChangefeed = 7, Unknown = Max(), }; diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp index e66ceb5d8a6e..14bb29dfee20 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp @@ -122,8 +122,6 @@ NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress:: return NImport::EImportProgress::Preparing; case Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA: return NImport::EImportProgress::TransferData; - case Ydb::Import::ImportProgress::PROGRESS_CREATE_CHANGEFEEDS: - return NImport::EImportProgress::CreateChangefeed; case Ydb::Import::ImportProgress::PROGRESS_DONE: return NImport::EImportProgress::Done; case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h index 1a45ddbb05d5..d4975e75620e 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/import/import.h @@ -17,7 +17,7 @@ enum class EImportProgress { Done = 4, Cancellation = 5, Cancelled = 6, - CreateChangefeed = 7, + CreateChangefeeds = 7, Unknown = std::numeric_limits::max(), }; From 9ce2215287646ad04fa6cfe97fc123d31de7db11 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Feb 2025 12:31:14 +0300 Subject: [PATCH 37/65] removed set flag --- ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 3 --- ydb/public/sdk/cpp/src/client/proto/accessor.cpp | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 1f36726005f7..876023a1ccbe 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1162,9 +1162,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase void OnNotifyResult(TTransactionContext& txc, const TActorContext& ctx) { Y_ABORT_UNLESS(CompletedTxId); - - AppData()->FeatureFlags.SetEnableChangefeedsImport(true); - LOG_D("TImport::TTxProgress: OnNotifyResult" << ": txId# " << CompletedTxId); diff --git a/ydb/public/sdk/cpp/src/client/proto/accessor.cpp b/ydb/public/sdk/cpp/src/client/proto/accessor.cpp index c34a33e8b939..fa85db31ae1f 100644 --- a/ydb/public/sdk/cpp/src/client/proto/accessor.cpp +++ b/ydb/public/sdk/cpp/src/client/proto/accessor.cpp @@ -125,7 +125,7 @@ NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress:: case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: return NImport::EImportProgress::BuildIndexes; case Ydb::Import::ImportProgress::PROGRESS_CREATE_CHANGEFEEDS: - return NImport::EImportProgress::CreateChangefeed; + return NImport::EImportProgress::CreateChangefeeds; case Ydb::Import::ImportProgress::PROGRESS_DONE: return NImport::EImportProgress::Done; case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: From 86c19bc7367835a7c5fdd3996fb7a87e367a68b4 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Feb 2025 12:45:32 +0300 Subject: [PATCH 38/65] add --- ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp index 14bb29dfee20..5f82e9236c14 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.cpp @@ -122,6 +122,8 @@ NImport::EImportProgress TProtoAccessor::FromProto(Ydb::Import::ImportProgress:: return NImport::EImportProgress::Preparing; case Ydb::Import::ImportProgress::PROGRESS_TRANSFER_DATA: return NImport::EImportProgress::TransferData; + case Ydb::Import::ImportProgress::PROGRESS_BUILD_INDEXES: + return NImport::EImportProgress::BuildIndexes; case Ydb::Import::ImportProgress::PROGRESS_DONE: return NImport::EImportProgress::Done; case Ydb::Import::ImportProgress::PROGRESS_CANCELLATION: From 90e0c8c88f3ad96a13b3d98065f8131e8fccf40e Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Feb 2025 13:27:12 +0300 Subject: [PATCH 39/65] fix --- .../schemeshard_import_scheme_getter.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 67a1353ca2d5..5e40ffe9fc6e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -338,13 +338,8 @@ class TSchemeGetter: public TActorBootstrapped { return Reply(false, "Cannot parse сhangefeed"); } - if (IndexDownloadedChangefeed == item.Changefeeds.ChangefeedsSize()) { - *item.Changefeeds.MutableChangefeeds()->Add()->MutableChangefeed() = std::move(changefeed); - } else { - *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableChangefeed() = std::move(changefeed); - } + *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableChangefeed() = std::move(changefeed); - auto nextStep = [this]() { Become(&TThis::StateDownloadTopics); HeadObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); @@ -401,10 +396,15 @@ class TSchemeGetter: public TActorBootstrapped { void ListObjects(const TString& prefix) { auto request = Model::ListObjectsRequest() .WithPrefix(prefix); - + Send(Client, new TEvExternalStorage::TEvListObjectsRequest(request)); } + template + void Resize(::google::protobuf::RepeatedPtrField* repeatedField, ui64 size) { + while (size--) repeatedField->Add(); + } + void HandleChangefeeds(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { const auto& result = ev.Get()->Get()->Result; LOG_D("HandleChangefeeds TEvExternalStorage::TEvListObjectResponse" @@ -427,6 +427,9 @@ class TSchemeGetter: public TActorBootstrapped { } if (!ChangefeedsKeys.empty()) { + auto& item = ImportInfo->Items.at(ItemIdx); + Resize(item.Changefeeds.MutableChangefeeds(), ChangefeedsKeys.size()); + Y_ABORT_UNLESS(IndexDownloadedChangefeed < ChangefeedsKeys.size()); HeadObject(ChangefeedDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); } else { From 1e205ac3edf34b99034241fe5012755eb6f075ff Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Feb 2025 13:31:50 +0300 Subject: [PATCH 40/65] removed --- ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 5e40ffe9fc6e..800b68472472 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -339,7 +339,7 @@ class TSchemeGetter: public TActorBootstrapped { } *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableChangefeed() = std::move(changefeed); - + auto nextStep = [this]() { Become(&TThis::StateDownloadTopics); HeadObject(TopicDescriptionKey(ChangefeedsKeys[IndexDownloadedChangefeed])); @@ -396,7 +396,7 @@ class TSchemeGetter: public TActorBootstrapped { void ListObjects(const TString& prefix) { auto request = Model::ListObjectsRequest() .WithPrefix(prefix); - + Send(Client, new TEvExternalStorage::TEvListObjectsRequest(request)); } From 8749ea1249beb5db08fb2ea595d541ebd3f3d2b6 Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Mon, 10 Feb 2025 15:34:25 +0300 Subject: [PATCH 41/65] Update ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp Co-authored-by: Ilnaz Nizametdinov --- ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index 800b68472472..ca32b0e198fa 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -401,7 +401,7 @@ class TSchemeGetter: public TActorBootstrapped { } template - void Resize(::google::protobuf::RepeatedPtrField* repeatedField, ui64 size) { + static void Resize(::google::protobuf::RepeatedPtrField* repeatedField, ui64 size) { while (size--) repeatedField->Add(); } From 39e7aacee9dd9c7378785209538d3a5fd598fe84 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 10 Feb 2025 16:20:31 +0300 Subject: [PATCH 42/65] removed test --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 9a34c94a1466..370a9e923a0b 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5115,34 +5115,4 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )" ); } - - // Y_UNIT_TEST(ShouldSucceedOnSingleShardTableWithChangefeed) { - // TTestBasicRuntime runtime; - - // const auto data = GenerateTestData(R"( - // columns { - // name: "key" - // type { optional_type { item { type_id: UTF8 } } } - // } - // columns { - // name: "value" - // type { optional_type { item { type_id: UTF8 } } } - // } - // primary_key: "key" - // )", {{"a", 1}}); - - // Run(runtime, ConvertTestData(data), R"( - // ImportFromS3Settings { - // endpoint: "localhost:%d" - // scheme: HTTP - // items { - // source_prefix: "" - // destination_path: "/MyRoot/Table" - // } - // } - // )"); - - // auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets); - // NKqp::CompareYson(data.Data[0].YsonStr, content); - // } } From a94309308945062bb199da53493f04f03bb36de3 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 11 Feb 2025 17:13:25 +0300 Subject: [PATCH 43/65] list objects in mock --- .../schemeshard_import__create.cpp | 1 + ydb/core/wrappers/ut_helpers/s3_mock.cpp | 66 ++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 876023a1ccbe..bc1d3f52fa3c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1161,6 +1161,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } void OnNotifyResult(TTransactionContext& txc, const TActorContext& ctx) { + AppData()->FeatureFlags.SetEnableChangefeedsImport(true); Y_ABORT_UNLESS(CompletedTxId); LOG_D("TImport::TTxProgress: OnNotifyResult" << ": txId# " << CompletedTxId); diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.cpp b/ydb/core/wrappers/ut_helpers/s3_mock.cpp index f9e30df5e0ba..cbd04f532653 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.cpp +++ b/ydb/core/wrappers/ut_helpers/s3_mock.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -152,6 +153,63 @@ bool TS3Mock::TRequest::HttpServeRead(const TReplyParams& params, EMethod method return true; } +TString BuildContentXML(const TString& path) { + return Sprintf(R"( + + %s + + )", path.c_str()); +} + +TString BuildContentListXML(const TVector& paths) { + TString result; + for (const auto& path : paths) { + result += BuildContentXML(path); + } + return result; +} + +TString BuildListObjectsXML(const TVector& paths, const TStringBuf bucketName) { + + return Sprintf(R"( + + + %s + false + %s + + )", bucketName.data(), BuildContentListXML(paths).c_str()); +} + +bool ServeList(const TRequestReplier::TReplyParams& params, const TString& prefix, const TStringBuf bucketName, const THashMap data) { + Cerr << "S3_MOCK::ServeList: " << prefix << Endl; + params.Output << "HTTP/1.1 200 Ok\r\n"; + TVector paths; + THttpHeaders headers; + + for (const auto& [key, value] : data) { + TFsPath path = key; + if (path.IsSubpathOf(TStringBuilder() << bucketName << "/" << prefix)) { + paths.push_back(path); + } + } + + TString xml = BuildListObjectsXML(paths, bucketName); + + headers.AddHeader("Content-Type", "application/xml"); + headers.AddHeader("Content-Length", xml.length()); + headers.OutTo(¶ms.Output); + + params.Output << xml; + params.Output << "\r\n"; + params.Output.Flush(); + + Cerr << xml.length() << Endl; + Cerr << xml << Endl; + + return true; +} + bool TS3Mock::TRequest::HttpServeWrite(const TReplyParams& params, TStringBuf path, const TCgiParameters& queryParams) { TString content; ui64 length; @@ -357,6 +415,10 @@ bool TS3Mock::TRequest::DoReply(const TReplyParams& params) { const EMethod method = ParseMethod(methodStr.data()); TStringBuf pathStr = uriStr.NextTok('?'); + Cerr << "requestString: " << requestString << Endl; + Cerr << "uriStr: " << uriStr << Endl; + Cerr << "pathStr: " << pathStr << Endl; + TCgiParameters queryParams; queryParams.ScanAddAll(uriStr); @@ -366,7 +428,9 @@ bool TS3Mock::TRequest::DoReply(const TReplyParams& params) { case EMethod::Head: case EMethod::Get: - if (Parent->Data.contains(pathStr)) { + if (queryParams.Has("prefix")) { + return ServeList(params, queryParams.Get("prefix"), pathStr, Parent->GetData()); + }else if (Parent->Data.contains(pathStr)) { return HttpServeRead(params, method, pathStr); } else { return HttpNotFound(params, "NoSuchKey"); From e4ac3c1bb4591e17c09aaa8499b64a356f32d31b Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 11 Feb 2025 17:15:11 +0300 Subject: [PATCH 44/65] py tests --- contrib/python/ydb/py2/ydb/import_client.py | 1 + ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/python/ydb/py2/ydb/import_client.py b/contrib/python/ydb/py2/ydb/import_client.py index a11d77a0c688..368f1d30579f 100644 --- a/contrib/python/ydb/py2/ydb/import_client.py +++ b/contrib/python/ydb/py2/ydb/import_client.py @@ -27,6 +27,7 @@ class ImportProgress(enum.IntEnum): DONE = 4 CANCELLATION = 5 CANCELLED = 6 + CREATE_CHANGEFEEDS = 7 def _initialize_progresses(): diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index bc1d3f52fa3c..876023a1ccbe 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -1161,7 +1161,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } void OnNotifyResult(TTransactionContext& txc, const TActorContext& ctx) { - AppData()->FeatureFlags.SetEnableChangefeedsImport(true); Y_ABORT_UNLESS(CompletedTxId); LOG_D("TImport::TTxProgress: OnNotifyResult" << ": txId# " << CompletedTxId); From b5b42b78c2b545cc69406fbdcf46880d7dc871e6 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 12 Feb 2025 11:54:14 +0300 Subject: [PATCH 45/65] py3 test --- contrib/python/ydb/py3/ydb/import_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/python/ydb/py3/ydb/import_client.py b/contrib/python/ydb/py3/ydb/import_client.py index 830f10c5bb6c..b69e04eb3247 100644 --- a/contrib/python/ydb/py3/ydb/import_client.py +++ b/contrib/python/ydb/py3/ydb/import_client.py @@ -28,6 +28,7 @@ class ImportProgress(enum.IntEnum): DONE = 4 CANCELLATION = 5 CANCELLED = 6 + CREATE_CHANGEFEEDS = 7 def _initialize_progresses(): From 1aaf7bc8ea4fba0341602738fc3af753f5aca7f3 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 12 Feb 2025 14:42:51 +0300 Subject: [PATCH 46/65] clean up s3_mock --- ydb/core/wrappers/ut_helpers/s3_mock.cpp | 21 +++++++-------------- ydb/core/wrappers/ut_helpers/s3_mock.h | 1 + 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.cpp b/ydb/core/wrappers/ut_helpers/s3_mock.cpp index cbd04f532653..63103eeeeaef 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.cpp +++ b/ydb/core/wrappers/ut_helpers/s3_mock.cpp @@ -155,9 +155,9 @@ bool TS3Mock::TRequest::HttpServeRead(const TReplyParams& params, EMethod method TString BuildContentXML(const TString& path) { return Sprintf(R"( - - %s - + + %s + )", path.c_str()); } @@ -181,13 +181,13 @@ TString BuildListObjectsXML(const TVector& paths, const TStringBuf buck )", bucketName.data(), BuildContentListXML(paths).c_str()); } -bool ServeList(const TRequestReplier::TReplyParams& params, const TString& prefix, const TStringBuf bucketName, const THashMap data) { +bool TS3Mock::TRequest::HttpServeList(const TReplyParams& params, TStringBuf bucketName, const TString& prefix) { Cerr << "S3_MOCK::ServeList: " << prefix << Endl; params.Output << "HTTP/1.1 200 Ok\r\n"; TVector paths; THttpHeaders headers; - for (const auto& [key, value] : data) { + for (const auto& [key, value] : Parent->Data) { TFsPath path = key; if (path.IsSubpathOf(TStringBuilder() << bucketName << "/" << prefix)) { paths.push_back(path); @@ -203,9 +203,6 @@ bool ServeList(const TRequestReplier::TReplyParams& params, const TString& prefi params.Output << xml; params.Output << "\r\n"; params.Output.Flush(); - - Cerr << xml.length() << Endl; - Cerr << xml << Endl; return true; } @@ -415,10 +412,6 @@ bool TS3Mock::TRequest::DoReply(const TReplyParams& params) { const EMethod method = ParseMethod(methodStr.data()); TStringBuf pathStr = uriStr.NextTok('?'); - Cerr << "requestString: " << requestString << Endl; - Cerr << "uriStr: " << uriStr << Endl; - Cerr << "pathStr: " << pathStr << Endl; - TCgiParameters queryParams; queryParams.ScanAddAll(uriStr); @@ -429,8 +422,8 @@ bool TS3Mock::TRequest::DoReply(const TReplyParams& params) { case EMethod::Head: case EMethod::Get: if (queryParams.Has("prefix")) { - return ServeList(params, queryParams.Get("prefix"), pathStr, Parent->GetData()); - }else if (Parent->Data.contains(pathStr)) { + return HttpServeList(params, pathStr, queryParams.Get("prefix")); + } else if (Parent->Data.contains(pathStr)) { return HttpServeRead(params, method, pathStr); } else { return HttpNotFound(params, "NoSuchKey"); diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.h b/ydb/core/wrappers/ut_helpers/s3_mock.h index c537fd09ef7c..8f7fa0e7275c 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.h +++ b/ydb/core/wrappers/ut_helpers/s3_mock.h @@ -47,6 +47,7 @@ class TS3Mock: public THttpServer::ICallBack { bool HttpNotImplemented(const TReplyParams& params); void MaybeContinue(const TReplyParams& params); bool HttpServeRead(const TReplyParams& params, EMethod method, const TStringBuf path); + bool HttpServeList(const TReplyParams& params, TStringBuf bucketName, const TString& prefix); bool HttpServeWrite(const TReplyParams& params, TStringBuf path, const TCgiParameters& queryParams); bool HttpServeAction(const TReplyParams& params, EMethod method, TStringBuf path, const TCgiParameters& queryParams); From e3b6729ff001e74bac423d685523c1bdf7993bb6 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Wed, 12 Feb 2025 16:01:22 +0300 Subject: [PATCH 47/65] prepare tests --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 370a9e923a0b..a36f19dbab8b 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -139,13 +139,19 @@ namespace { } }; + struct TImportChangefeed { + const TString ChangefeedName; + const TString Changefeed; + const TString Topic; + }; + struct TTestDataWithScheme { TString Metadata; EPathType Type = EPathTypeTable; TString Scheme; TString CreationQuery; TString Permissions; - TVector Changefeeds; + TVector Changefeeds; TVector Data; TTestDataWithScheme() = default; @@ -253,7 +259,7 @@ namespace { const TVector>& shardsConfig = {{"a", 1}}, const TString& permissions = "", const TString& metadata = "", - const TVector changefeeds = {} + const TVector changefeeds = {} ) { TTestDataWithScheme result; result.Type = typedScheme.Type; @@ -304,6 +310,13 @@ namespace { if (item.Permissions) { result.emplace(prefix + "/permissions.pb", item.Permissions); } + + for (const auto& importChangefeed : item.Changefeeds) { + const TString newPrefix = TStringBuilder() << prefix << "/" << importChangefeed.ChangefeedName; + result.emplace(newPrefix + "/changefeed_description.pb", importChangefeed.Changefeed); + result.emplace(newPrefix + "/topic_description.pb", importChangefeed.Topic); + } + for (ui32 i = 0; i < item.Data.size(); ++i) { const auto& data = item.Data.at(i); result.emplace(Sprintf("%s/data_%02d%s", prefix.data(), i, data.Ext().c_str()), data.Data); @@ -5115,4 +5128,50 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )" ); } + + Y_UNIT_TEST(Changefeeds) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + // const auto changefeedDesc = ; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::PathExist, + NLs::HasOwner("eve"), + NLs::HasRight("+R:alice"), + NLs::HasRight("+W:alice"), + NLs::HasRight("+R:bob") + }); + } } From f524e3022ed583840a09c84f7bd8661636f219f7 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 13 Feb 2025 11:25:48 +0300 Subject: [PATCH 48/65] try/except in py --- contrib/python/ydb/py2/ydb/import_client.py | 6 ++++-- contrib/python/ydb/py3/ydb/import_client.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contrib/python/ydb/py2/ydb/import_client.py b/contrib/python/ydb/py2/ydb/import_client.py index 368f1d30579f..851aa392d366 100644 --- a/contrib/python/ydb/py2/ydb/import_client.py +++ b/contrib/python/ydb/py2/ydb/import_client.py @@ -27,12 +27,14 @@ class ImportProgress(enum.IntEnum): DONE = 4 CANCELLATION = 5 CANCELLED = 6 - CREATE_CHANGEFEEDS = 7 def _initialize_progresses(): for key, value in ydb_import_pb2.ImportProgress.Progress.items(): - _progresses[value] = getattr(ImportProgress, key[len("PROGRESS_") :]) + try: + _progresses[value] = getattr(ImportProgress, key[len("PROGRESS_") :]) + except AttributeError: + pass _initialize_progresses() diff --git a/contrib/python/ydb/py3/ydb/import_client.py b/contrib/python/ydb/py3/ydb/import_client.py index b69e04eb3247..9a01e5a508bc 100644 --- a/contrib/python/ydb/py3/ydb/import_client.py +++ b/contrib/python/ydb/py3/ydb/import_client.py @@ -28,12 +28,14 @@ class ImportProgress(enum.IntEnum): DONE = 4 CANCELLATION = 5 CANCELLED = 6 - CREATE_CHANGEFEEDS = 7 def _initialize_progresses(): for key, value in ydb_import_pb2.ImportProgress.Progress.items(): - _progresses[value] = getattr(ImportProgress, key[len("PROGRESS_") :]) + try: + _progresses[value] = getattr(ImportProgress, key[len("PROGRESS_") :]) + except AttributeError: + pass _initialize_progresses() From 8c66604b0f542f2d0d6f1e9f6e05dca51d3d84a7 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 13 Feb 2025 11:40:40 +0300 Subject: [PATCH 49/65] test removed --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 92 ++++++++++--------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index a36f19dbab8b..26f34b119e0c 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5129,49 +5129,51 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { ); } - Y_UNIT_TEST(Changefeeds) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - - // const auto changefeedDesc = ; - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 1}}, permissions); - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", port)); - env.TestWaitNotification(runtime, txId); - - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { - NLs::PathExist, - NLs::HasOwner("eve"), - NLs::HasRight("+R:alice"), - NLs::HasRight("+W:alice"), - NLs::HasRight("+R:bob") - }); - } + // Y_UNIT_TEST(Changefeeds) { + // TTestBasicRuntime runtime; + // TTestEnv env(runtime); + // ui64 txId = 100; + + // const auto changefeedDesc = R"( + + // )"; + + // const auto data = GenerateTestData(R"( + // columns { + // name: "key" + // type { optional_type { item { type_id: UTF8 } } } + // } + // columns { + // name: "value" + // type { optional_type { item { type_id: UTF8 } } } + // } + // primary_key: "key" + // )", {{"a", 1}}); + + // TPortManager portManager; + // const ui16 port = portManager.GetPort(); + + // TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + // UNIT_ASSERT(s3Mock.Start()); + + // TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + // ImportFromS3Settings { + // endpoint: "localhost:%d" + // scheme: HTTP + // items { + // source_prefix: "" + // destination_path: "/MyRoot/Table" + // } + // } + // )", port)); + // env.TestWaitNotification(runtime, txId); + + // TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + // NLs::PathExist, + // NLs::HasOwner("eve"), + // NLs::HasRight("+R:alice"), + // NLs::HasRight("+W:alice"), + // NLs::HasRight("+R:bob") + // }); + // } } From 6c58e40c11c8e215c0a0ef45bce2a9212a5e5032 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 13 Feb 2025 13:20:50 +0300 Subject: [PATCH 50/65] fix build ut_restore --- ydb/core/tx/schemeshard/schemeshard_export__create.cpp | 1 + ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index 0b456e3f1f87..7e7eb5935233 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -1026,6 +1026,7 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase } void OnNotifyResult(TTransactionContext& txc, const TActorContext& ctx) { + AppData()->FeatureFlags.SetEnableChangefeedsImport(true); Y_ABORT_UNLESS(CompletedTxId); LOG_D("TExport::TTxProgress: OnNotifyResult" << ": txId# " << CompletedTxId); diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 26f34b119e0c..150661398953 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -140,9 +140,9 @@ namespace { }; struct TImportChangefeed { - const TString ChangefeedName; - const TString Changefeed; - const TString Topic; + TString ChangefeedName; + TString Changefeed; + TString Topic; }; struct TTestDataWithScheme { @@ -259,13 +259,13 @@ namespace { const TVector>& shardsConfig = {{"a", 1}}, const TString& permissions = "", const TString& metadata = "", - const TVector changefeeds = {} + TVector&& changefeeds = {} ) { TTestDataWithScheme result; result.Type = typedScheme.Type; result.Permissions = permissions; result.Metadata = metadata; - result.Changefeeds = changefeeds; + result.Changefeeds = std::move(changefeeds); switch (typedScheme.Type) { case EPathTypeTable: From f777f6f0d2c974e37862ff024cac6545043546f9 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Thu, 13 Feb 2025 14:39:24 +0300 Subject: [PATCH 51/65] try test --- .../tx/schemeshard/ut_helpers/ls_checks.cpp | 6 + .../tx/schemeshard/ut_helpers/ls_checks.h | 1 + .../tx/schemeshard/ut_restore/ut_restore.cpp | 151 ++++++++++++------ 3 files changed, 111 insertions(+), 47 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 25804068e46c..f0244fa20ff4 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -917,6 +917,12 @@ TCheckFunc SequenceCache(ui64 cache) { }; } +TCheckFunc StreamName(const TString& name) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetCdcStreamDescription().GetName(), name); + }; +} + TCheckFunc StreamMode(NKikimrSchemeOp::ECdcStreamMode mode) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetCdcStreamDescription().GetMode(), mode); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 1813edca62b5..d8440bd5501f 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -162,6 +162,7 @@ namespace NLs { TCheckFunc SequenceCache(ui64 cache); TCheckFunc StreamMode(NKikimrSchemeOp::ECdcStreamMode mode); + TCheckFunc StreamName(const TString& name); TCheckFunc StreamFormat(NKikimrSchemeOp::ECdcStreamFormat format); TCheckFunc StreamState(NKikimrSchemeOp::ECdcStreamState state); TCheckFunc StreamVirtualTimestamps(bool value); diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 150661398953..633b69caff85 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -321,6 +321,11 @@ namespace { const auto& data = item.Data.at(i); result.emplace(Sprintf("%s/data_%02d%s", prefix.data(), i, data.Ext().c_str()), data.Data); } + + // for (const auto& x : result) { + // Cerr << "key: " << x.first << Endl << "value: " << x.second << Endl; + // } + } return result; @@ -5129,51 +5134,103 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { ); } - // Y_UNIT_TEST(Changefeeds) { - // TTestBasicRuntime runtime; - // TTestEnv env(runtime); - // ui64 txId = 100; - - // const auto changefeedDesc = R"( - - // )"; - - // const auto data = GenerateTestData(R"( - // columns { - // name: "key" - // type { optional_type { item { type_id: UTF8 } } } - // } - // columns { - // name: "value" - // type { optional_type { item { type_id: UTF8 } } } - // } - // primary_key: "key" - // )", {{"a", 1}}); - - // TPortManager portManager; - // const ui16 port = portManager.GetPort(); - - // TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); - // UNIT_ASSERT(s3Mock.Start()); - - // TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( - // ImportFromS3Settings { - // endpoint: "localhost:%d" - // scheme: HTTP - // items { - // source_prefix: "" - // destination_path: "/MyRoot/Table" - // } - // } - // )", port)); - // env.TestWaitNotification(runtime, txId); - - // TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { - // NLs::PathExist, - // NLs::HasOwner("eve"), - // NLs::HasRight("+R:alice"), - // NLs::HasRight("+W:alice"), - // NLs::HasRight("+R:bob") - // }); - // } + Y_UNIT_TEST(Changefeeds) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + const auto changefeedDesc = R"( + name: "updates_feed1" + mode: MODE_UPDATES + format: FORMAT_JSON + state: STATE_ENABLED + )"; + + const auto topicDesc = R"( + partitioning_settings { + min_active_partitions: 1 + max_active_partitions: 1 + auto_partitioning_settings { + strategy: AUTO_PARTITIONING_STRATEGY_DISABLED + partition_write_speed { + stabilization_window { + seconds: 300 + } + up_utilization_percent: 80 + down_utilization_percent: 20 + } + } + } + partitions { + active: true + } + retention_period { + seconds: 86400 + } + partition_write_speed_bytes_per_second: 1048576 + partition_write_burst_bytes: 1048576 + attributes { + key: "__max_partition_message_groups_seqno_stored" + value: "6000000" + } + attributes { + key: "_allow_unauthenticated_read" + value: "true" + } + attributes { + key: "_allow_unauthenticated_write" + value: "true" + } + attributes { + key: "_message_group_seqno_retention_period_ms" + value: "1382400000" + } + consumers { + name: "my_consumer" + read_from { + } + attributes { + key: "_service_type" + value: "data-streams" + } + } + )"; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, "", "", {{"updates_feed1", changefeedDesc, topicDesc}}); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::PathExist, + }); + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table/updates_feed1"), { + NLs::PathExist, + }); + } } From 2684d4ad33e5b3ccbc52d3db7644e4ae9e84bb05 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 14 Feb 2025 20:23:44 +0300 Subject: [PATCH 52/65] tests --- .../schemeshard_import_scheme_getter.cpp | 2 +- .../ut_helpers/export_reboots_common.cpp | 2 +- .../ut_helpers/ut_backup_restore_common.h | 23 ++++ .../tx/schemeshard/ut_restore/ut_restore.cpp | 104 +++++++++++++----- 4 files changed, 100 insertions(+), 31 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index ca32b0e198fa..1a311fb5262c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -111,7 +111,7 @@ class TSchemeGetter: public TActorBootstrapped { << ", result# " << result); if (NoObjectFound(result.GetError().GetErrorType())) { - Reply(); // permissions are optional + StartDownloadingChangefeeds(); // permissions are optional return; } else if (!CheckResult(result, "HeadObject")) { return; diff --git a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp index 899620a2d2e0..a918ed04d6f3 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/export_reboots_common.cpp @@ -13,7 +13,7 @@ namespace NExportReboots { void CreateSchemeObjects(TTestWithReboots& t, TTestActorRuntime& runtime, const TVector& schemeObjects) { TSet toWait; - for (const auto& [type, scheme] : schemeObjects) { + for (const auto& [type, scheme, _] : schemeObjects) { switch (type) { case EPathTypeTable: TestCreateTable(runtime, ++t.TxId, "/MyRoot", scheme); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h b/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h index eec5d324497c..1be58e907d96 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h @@ -1,6 +1,8 @@ #include #include +#include + using EDataFormat = NKikimr::NDataShard::NBackupRestoreTraits::EDataFormat; using ECompressionCodec = NKikimr::NDataShard::NBackupRestoreTraits::ECompressionCodec; @@ -16,9 +18,24 @@ using ECompressionCodec = NKikimr::NDataShard::NBackupRestoreTraits::ECompressio template \ void N(NUnitTest::TTestContext&) +namespace NAttr { + +enum class EKeys { + TOPIC_DESCRIPTION, +}; +class TAttributes : public THashMap { +public: + const TString& GetTopicDescription() const { + return this->at(EKeys::TOPIC_DESCRIPTION); + } + +}; +} // NAttr + struct TTypedScheme { NKikimrSchemeOp::EPathType Type; TString Scheme; + NAttr::TAttributes Attributes; TTypedScheme(const char* scheme) : Type(NKikimrSchemeOp::EPathTypeTable) @@ -34,4 +51,10 @@ struct TTypedScheme { : Type(type) , Scheme(std::move(scheme)) {} + + TTypedScheme(NKikimrSchemeOp::EPathType type, TString scheme, NAttr::TAttributes&& attributes) + : Type(type) + , Scheme(std::move(scheme)) + , Attributes(std::move(attributes)) + {} }; diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 633b69caff85..3602fcc500a1 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -140,7 +140,6 @@ namespace { }; struct TImportChangefeed { - TString ChangefeedName; TString Changefeed; TString Topic; }; @@ -151,7 +150,7 @@ namespace { TString Scheme; TString CreationQuery; TString Permissions; - TVector Changefeeds; + TImportChangefeed Changefeed; TVector Data; TTestDataWithScheme() = default; @@ -258,14 +257,12 @@ namespace { const TTypedScheme& typedScheme, const TVector>& shardsConfig = {{"a", 1}}, const TString& permissions = "", - const TString& metadata = "", - TVector&& changefeeds = {} + const TString& metadata = "" ) { TTestDataWithScheme result; result.Type = typedScheme.Type; result.Permissions = permissions; result.Metadata = metadata; - result.Changefeeds = std::move(changefeeds); switch (typedScheme.Type) { case EPathTypeTable: @@ -277,6 +274,10 @@ namespace { case EPathTypeView: result.CreationQuery = typedScheme.Scheme; break; + case EPathTypeCdcStream: + Cerr << "EPathTypeCdcStream23434" << Endl; + result.Changefeed = {typedScheme.Scheme, typedScheme.Attributes.GetTopicDescription()}; + break; default: UNIT_FAIL("cannot create sample test data for the scheme object type: " << typedScheme.Type); return {}; @@ -296,6 +297,10 @@ namespace { case EPathTypeView: result.emplace(prefix + "/create_view.sql", item.CreationQuery); break; + case EPathTypeCdcStream: + result.emplace(prefix + "/changefeed_description.pb", item.Changefeed.Changefeed); + result.emplace(prefix + "/topic_description.pb", item.Changefeed.Topic); + break; default: UNIT_FAIL("cannot determine key for the scheme object type: " << item.Type); return {}; @@ -311,21 +316,10 @@ namespace { result.emplace(prefix + "/permissions.pb", item.Permissions); } - for (const auto& importChangefeed : item.Changefeeds) { - const TString newPrefix = TStringBuilder() << prefix << "/" << importChangefeed.ChangefeedName; - result.emplace(newPrefix + "/changefeed_description.pb", importChangefeed.Changefeed); - result.emplace(newPrefix + "/topic_description.pb", importChangefeed.Topic); - } - for (ui32 i = 0; i < item.Data.size(); ++i) { const auto& data = item.Data.at(i); result.emplace(Sprintf("%s/data_%02d%s", prefix.data(), i, data.Ext().c_str()), data.Data); } - - // for (const auto& x : result) { - // Cerr << "key: " << x.first << Endl << "value: " << x.second << Endl; - // } - } return result; @@ -5134,17 +5128,21 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { ); } - Y_UNIT_TEST(Changefeeds) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; + struct GeneratedChangefeed { + std::pair Changefeed; + std::function Checker; + }; - const auto changefeedDesc = R"( - name: "updates_feed1" + GeneratedChangefeed GenChangefeed(ui64 num = 1) { + const TString changefeedName = TStringBuilder() << "updates_feed" << num; + const auto changefeedPath = TStringBuilder() << "/" << changefeedName; + + const auto changefeedDesc = Sprintf(R"( + name: "%s" mode: MODE_UPDATES format: FORMAT_JSON state: STATE_ENABLED - )"; + )", changefeedName.c_str()); const auto topicDesc = R"( partitioning_settings { @@ -5196,6 +5194,42 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { } )"; + NAttr::TAttributes attr; + attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc); + return { + {changefeedPath, GenerateTestData( + { + EPathTypeCdcStream, + changefeedDesc, + std::move(attr) + } + )}, + [changefeedPath = TString(changefeedPath)](TTestBasicRuntime& runtime){ + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table" + changefeedPath, false, false, true), { + NLs::PathExist + }); + } + }; + } + + TVector > GenChangefeeds(THashMap& bucketContent, ui64 count = 1) { + TVector> checkers; + checkers.reserve(count); + for (ui64 i = 1; i <= count; ++i) { + auto genChangefeed = GenChangefeed(i); + bucketContent.emplace(genChangefeed.Changefeed); + checkers.push_back(genChangefeed.Checker); + } + return checkers; + } + + void TestImportChangefeeds(ui64 countChangefeed) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + runtime.GetAppData().FeatureFlags.SetEnableChangefeedsImport(true); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + const auto data = GenerateTestData(R"( columns { name: "key" @@ -5206,12 +5240,16 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { type { optional_type { item { type_id: UTF8 } } } } primary_key: "key" - )", {{"a", 1}}, "", "", {{"updates_feed1", changefeedDesc, topicDesc}}); + )"); + + THashMap bucketContent(countChangefeed + 1); + bucketContent.emplace("", data); + auto checkers = GenChangefeeds(bucketContent, countChangefeed); TPortManager portManager; const ui16 port = portManager.GetPort(); - TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + TS3Mock s3Mock(ConvertTestData(bucketContent), TS3Mock::TSettings(port)); UNIT_ASSERT(s3Mock.Start()); TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( @@ -5227,10 +5265,18 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { env.TestWaitNotification(runtime, txId); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { - NLs::PathExist, - }); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table/updates_feed1"), { - NLs::PathExist, + NLs::PathExist }); + for (const auto& checker : checkers) { + checker(runtime); + } + } + + Y_UNIT_TEST(Changefeed) { + TestImportChangefeeds(1); + } + + Y_UNIT_TEST(Changefeeds) { + TestImportChangefeeds(3); } } From 2bc46f5bef3275e6b50c96db720d8f25e1f69bf2 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 14 Feb 2025 20:26:13 +0300 Subject: [PATCH 53/65] removed extra --- ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp | 6 ------ ydb/core/tx/schemeshard/ut_helpers/ls_checks.h | 1 - 2 files changed, 7 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index f0244fa20ff4..25804068e46c 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -917,12 +917,6 @@ TCheckFunc SequenceCache(ui64 cache) { }; } -TCheckFunc StreamName(const TString& name) { - return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { - UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetCdcStreamDescription().GetName(), name); - }; -} - TCheckFunc StreamMode(NKikimrSchemeOp::ECdcStreamMode mode) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { UNIT_ASSERT_VALUES_EQUAL(record.GetPathDescription().GetCdcStreamDescription().GetMode(), mode); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index d8440bd5501f..1813edca62b5 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -162,7 +162,6 @@ namespace NLs { TCheckFunc SequenceCache(ui64 cache); TCheckFunc StreamMode(NKikimrSchemeOp::ECdcStreamMode mode); - TCheckFunc StreamName(const TString& name); TCheckFunc StreamFormat(NKikimrSchemeOp::ECdcStreamFormat format); TCheckFunc StreamState(NKikimrSchemeOp::ECdcStreamState state); TCheckFunc StreamVirtualTimestamps(bool value); From 7dbb241a44adb9127e49f0348423b5e79fee7093 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 14 Feb 2025 20:28:31 +0300 Subject: [PATCH 54/65] removed cerr --- ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 3602fcc500a1..6bdb8083e234 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -275,7 +275,6 @@ namespace { result.CreationQuery = typedScheme.Scheme; break; case EPathTypeCdcStream: - Cerr << "EPathTypeCdcStream23434" << Endl; result.Changefeed = {typedScheme.Scheme, typedScheme.Attributes.GetTopicDescription()}; break; default: From bbf1f4f1e62b6a3e3f3b3d6ce4397183474dd1ed Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 14 Feb 2025 22:00:12 +0300 Subject: [PATCH 55/65] tests with permissions --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 97 +++++++++++++++++-- 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 6bdb8083e234..0ee681f999cd 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5211,7 +5211,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { }; } - TVector > GenChangefeeds(THashMap& bucketContent, ui64 count = 1) { + TVector> GenChangefeeds(THashMap& bucketContent, ui64 count = 1) { TVector> checkers; checkers.reserve(count); for (ui64 i = 1; i <= count; ++i) { @@ -5222,7 +5222,61 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { return checkers; } - void TestImportChangefeeds(ui64 countChangefeed) { + std::function AddedSchemeCommon(THashMap& bucketContent, const TString& permissions) { + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + bucketContent.emplace("", data); + return [](TTestBasicRuntime& runtime){ + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::PathExist + }); + }; + } + + std::function AddedScheme(THashMap& bucketContent) { + return AddedSchemeCommon(bucketContent, ""); + } + + std::function AddedSchemeWithPermissions(THashMap& bucketContent) { + const auto permissions = R"( + actions { + change_owner: "eve" + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.read" + } + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.write" + } + } + actions { + grant { + subject: "bob" + permission_names: "ydb.generic.read" + } + } + )"; + return AddedSchemeCommon(bucketContent, permissions); + } + + using SchemeFunction = std::function(THashMap&)>; + + void TestImportChangefeeds(ui64 countChangefeed, SchemeFunction addedScheme) { TTestBasicRuntime runtime; TTestEnv env(runtime); ui64 txId = 100; @@ -5242,8 +5296,9 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )"); THashMap bucketContent(countChangefeed + 1); - bucketContent.emplace("", data); - auto checkers = GenChangefeeds(bucketContent, countChangefeed); + + auto checkerTable = addedScheme(bucketContent); + auto checkersChangefeeds = GenChangefeeds(bucketContent, countChangefeed); TPortManager portManager; const ui16 port = portManager.GetPort(); @@ -5263,19 +5318,41 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )", port)); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { - NLs::PathExist - }); - for (const auto& checker : checkers) { + checkerTable(runtime); + for (const auto& checker : checkersChangefeeds) { checker(runtime); } } + void TestImportChangefeed() { + TestImportChangefeeds(1, AddedScheme); + } + + void TestImportChangefeeds() { + TestImportChangefeeds(3, AddedScheme); + } + + void TestImportChangefeedWithTablePermissions() { + TestImportChangefeeds(1, AddedSchemeWithPermissions); + } + + void TestImportChangefeedsWithTablePermissions() { + TestImportChangefeeds(3, AddedSchemeWithPermissions); + } + Y_UNIT_TEST(Changefeed) { - TestImportChangefeeds(1); + TestImportChangefeed(); } Y_UNIT_TEST(Changefeeds) { - TestImportChangefeeds(3); + TestImportChangefeeds(); + } + + Y_UNIT_TEST(ChangefeedWithTablePermissions) { + TestImportChangefeedWithTablePermissions(); + } + + Y_UNIT_TEST(ChangefeedsWithTablePermissions) { + TestImportChangefeedsWithTablePermissions(); } } From 163c20908655604a67a5bc14d10366639d8306bc Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Fri, 14 Feb 2025 22:16:21 +0300 Subject: [PATCH 56/65] new place for tests --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 458 +++++++++--------- 1 file changed, 229 insertions(+), 229 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 0ee681f999cd..3480af83dd56 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -4824,6 +4824,235 @@ Y_UNIT_TEST_SUITE(TImportTests) { NLs::IsView }); } + + struct GeneratedChangefeed { + std::pair Changefeed; + std::function Checker; + }; + + GeneratedChangefeed GenChangefeed(ui64 num = 1) { + const TString changefeedName = TStringBuilder() << "updates_feed" << num; + const auto changefeedPath = TStringBuilder() << "/" << changefeedName; + + const auto changefeedDesc = Sprintf(R"( + name: "%s" + mode: MODE_UPDATES + format: FORMAT_JSON + state: STATE_ENABLED + )", changefeedName.c_str()); + + const auto topicDesc = R"( + partitioning_settings { + min_active_partitions: 1 + max_active_partitions: 1 + auto_partitioning_settings { + strategy: AUTO_PARTITIONING_STRATEGY_DISABLED + partition_write_speed { + stabilization_window { + seconds: 300 + } + up_utilization_percent: 80 + down_utilization_percent: 20 + } + } + } + partitions { + active: true + } + retention_period { + seconds: 86400 + } + partition_write_speed_bytes_per_second: 1048576 + partition_write_burst_bytes: 1048576 + attributes { + key: "__max_partition_message_groups_seqno_stored" + value: "6000000" + } + attributes { + key: "_allow_unauthenticated_read" + value: "true" + } + attributes { + key: "_allow_unauthenticated_write" + value: "true" + } + attributes { + key: "_message_group_seqno_retention_period_ms" + value: "1382400000" + } + consumers { + name: "my_consumer" + read_from { + } + attributes { + key: "_service_type" + value: "data-streams" + } + } + )"; + + NAttr::TAttributes attr; + attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc); + return { + {changefeedPath, GenerateTestData( + { + EPathTypeCdcStream, + changefeedDesc, + std::move(attr) + } + )}, + [changefeedPath = TString(changefeedPath)](TTestBasicRuntime& runtime){ + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table" + changefeedPath, false, false, true), { + NLs::PathExist + }); + } + }; + } + + TVector> GenChangefeeds(THashMap& bucketContent, ui64 count = 1) { + TVector> checkers; + checkers.reserve(count); + for (ui64 i = 1; i <= count; ++i) { + auto genChangefeed = GenChangefeed(i); + bucketContent.emplace(genChangefeed.Changefeed); + checkers.push_back(genChangefeed.Checker); + } + return checkers; + } + + std::function AddedSchemeCommon(THashMap& bucketContent, const TString& permissions) { + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + bucketContent.emplace("", data); + return [](TTestBasicRuntime& runtime){ + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::PathExist + }); + }; + } + + std::function AddedScheme(THashMap& bucketContent) { + return AddedSchemeCommon(bucketContent, ""); + } + + std::function AddedSchemeWithPermissions(THashMap& bucketContent) { + const auto permissions = R"( + actions { + change_owner: "eve" + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.read" + } + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.write" + } + } + actions { + grant { + subject: "bob" + permission_names: "ydb.generic.read" + } + } + )"; + return AddedSchemeCommon(bucketContent, permissions); + } + + using SchemeFunction = std::function(THashMap&)>; + + void TestImportChangefeeds(ui64 countChangefeed, SchemeFunction addedScheme) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + runtime.GetAppData().FeatureFlags.SetEnableChangefeedsImport(true); + runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )"); + + THashMap bucketContent(countChangefeed + 1); + + auto checkerTable = addedScheme(bucketContent); + auto checkersChangefeeds = GenChangefeeds(bucketContent, countChangefeed); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(bucketContent), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + + checkerTable(runtime); + for (const auto& checker : checkersChangefeeds) { + checker(runtime); + } + } + + void TestImportChangefeed() { + TestImportChangefeeds(1, AddedScheme); + } + + void TestImportChangefeeds() { + TestImportChangefeeds(3, AddedScheme); + } + + void TestImportChangefeedWithTablePermissions() { + TestImportChangefeeds(1, AddedSchemeWithPermissions); + } + + void TestImportChangefeedsWithTablePermissions() { + TestImportChangefeeds(3, AddedSchemeWithPermissions); + } + + Y_UNIT_TEST(Changefeed) { + TestImportChangefeed(); + } + + Y_UNIT_TEST(Changefeeds) { + TestImportChangefeeds(); + } + + Y_UNIT_TEST(ChangefeedWithTablePermissions) { + TestImportChangefeedWithTablePermissions(); + } + + Y_UNIT_TEST(ChangefeedsWithTablePermissions) { + TestImportChangefeedsWithTablePermissions(); + } } Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { @@ -5126,233 +5355,4 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )" ); } - - struct GeneratedChangefeed { - std::pair Changefeed; - std::function Checker; - }; - - GeneratedChangefeed GenChangefeed(ui64 num = 1) { - const TString changefeedName = TStringBuilder() << "updates_feed" << num; - const auto changefeedPath = TStringBuilder() << "/" << changefeedName; - - const auto changefeedDesc = Sprintf(R"( - name: "%s" - mode: MODE_UPDATES - format: FORMAT_JSON - state: STATE_ENABLED - )", changefeedName.c_str()); - - const auto topicDesc = R"( - partitioning_settings { - min_active_partitions: 1 - max_active_partitions: 1 - auto_partitioning_settings { - strategy: AUTO_PARTITIONING_STRATEGY_DISABLED - partition_write_speed { - stabilization_window { - seconds: 300 - } - up_utilization_percent: 80 - down_utilization_percent: 20 - } - } - } - partitions { - active: true - } - retention_period { - seconds: 86400 - } - partition_write_speed_bytes_per_second: 1048576 - partition_write_burst_bytes: 1048576 - attributes { - key: "__max_partition_message_groups_seqno_stored" - value: "6000000" - } - attributes { - key: "_allow_unauthenticated_read" - value: "true" - } - attributes { - key: "_allow_unauthenticated_write" - value: "true" - } - attributes { - key: "_message_group_seqno_retention_period_ms" - value: "1382400000" - } - consumers { - name: "my_consumer" - read_from { - } - attributes { - key: "_service_type" - value: "data-streams" - } - } - )"; - - NAttr::TAttributes attr; - attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc); - return { - {changefeedPath, GenerateTestData( - { - EPathTypeCdcStream, - changefeedDesc, - std::move(attr) - } - )}, - [changefeedPath = TString(changefeedPath)](TTestBasicRuntime& runtime){ - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table" + changefeedPath, false, false, true), { - NLs::PathExist - }); - } - }; - } - - TVector> GenChangefeeds(THashMap& bucketContent, ui64 count = 1) { - TVector> checkers; - checkers.reserve(count); - for (ui64 i = 1; i <= count; ++i) { - auto genChangefeed = GenChangefeed(i); - bucketContent.emplace(genChangefeed.Changefeed); - checkers.push_back(genChangefeed.Checker); - } - return checkers; - } - - std::function AddedSchemeCommon(THashMap& bucketContent, const TString& permissions) { - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )", {{"a", 1}}, permissions); - - bucketContent.emplace("", data); - return [](TTestBasicRuntime& runtime){ - TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { - NLs::PathExist - }); - }; - } - - std::function AddedScheme(THashMap& bucketContent) { - return AddedSchemeCommon(bucketContent, ""); - } - - std::function AddedSchemeWithPermissions(THashMap& bucketContent) { - const auto permissions = R"( - actions { - change_owner: "eve" - } - actions { - grant { - subject: "alice" - permission_names: "ydb.generic.read" - } - } - actions { - grant { - subject: "alice" - permission_names: "ydb.generic.write" - } - } - actions { - grant { - subject: "bob" - permission_names: "ydb.generic.read" - } - } - )"; - return AddedSchemeCommon(bucketContent, permissions); - } - - using SchemeFunction = std::function(THashMap&)>; - - void TestImportChangefeeds(ui64 countChangefeed, SchemeFunction addedScheme) { - TTestBasicRuntime runtime; - TTestEnv env(runtime); - ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableChangefeedsImport(true); - runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); - - const auto data = GenerateTestData(R"( - columns { - name: "key" - type { optional_type { item { type_id: UTF8 } } } - } - columns { - name: "value" - type { optional_type { item { type_id: UTF8 } } } - } - primary_key: "key" - )"); - - THashMap bucketContent(countChangefeed + 1); - - auto checkerTable = addedScheme(bucketContent); - auto checkersChangefeeds = GenChangefeeds(bucketContent, countChangefeed); - - TPortManager portManager; - const ui16 port = portManager.GetPort(); - - TS3Mock s3Mock(ConvertTestData(bucketContent), TS3Mock::TSettings(port)); - UNIT_ASSERT(s3Mock.Start()); - - TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( - ImportFromS3Settings { - endpoint: "localhost:%d" - scheme: HTTP - items { - source_prefix: "" - destination_path: "/MyRoot/Table" - } - } - )", port)); - env.TestWaitNotification(runtime, txId); - - checkerTable(runtime); - for (const auto& checker : checkersChangefeeds) { - checker(runtime); - } - } - - void TestImportChangefeed() { - TestImportChangefeeds(1, AddedScheme); - } - - void TestImportChangefeeds() { - TestImportChangefeeds(3, AddedScheme); - } - - void TestImportChangefeedWithTablePermissions() { - TestImportChangefeeds(1, AddedSchemeWithPermissions); - } - - void TestImportChangefeedsWithTablePermissions() { - TestImportChangefeeds(3, AddedSchemeWithPermissions); - } - - Y_UNIT_TEST(Changefeed) { - TestImportChangefeed(); - } - - Y_UNIT_TEST(Changefeeds) { - TestImportChangefeeds(); - } - - Y_UNIT_TEST(ChangefeedWithTablePermissions) { - TestImportChangefeedWithTablePermissions(); - } - - Y_UNIT_TEST(ChangefeedsWithTablePermissions) { - TestImportChangefeedsWithTablePermissions(); - } } From e4117481573cdcd541785b50ec278f3b6ef80a10 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 17 Feb 2025 12:59:59 +0300 Subject: [PATCH 57/65] fix logs --- ydb/core/wrappers/ut_helpers/s3_mock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.cpp b/ydb/core/wrappers/ut_helpers/s3_mock.cpp index 63103eeeeaef..0a620fdc2714 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.cpp +++ b/ydb/core/wrappers/ut_helpers/s3_mock.cpp @@ -182,7 +182,7 @@ TString BuildListObjectsXML(const TVector& paths, const TStringBuf buck } bool TS3Mock::TRequest::HttpServeList(const TReplyParams& params, TStringBuf bucketName, const TString& prefix) { - Cerr << "S3_MOCK::ServeList: " << prefix << Endl; + Cerr << "S3_MOCK::HttpServeList: " << prefix << Endl; params.Output << "HTTP/1.1 200 Ok\r\n"; TVector paths; THttpHeaders headers; From c3cb505b3dc35b756ac442cbc2381e5ba920c094 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Mon, 17 Feb 2025 21:37:14 +0300 Subject: [PATCH 58/65] test with reboots --- .../schemeshard_import__create.cpp | 13 ++- .../tx/schemeshard/ut_restore/ut_restore.cpp | 89 +++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index fb539b8e8767..ed53ff086622 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -554,7 +554,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); const auto& item = importInfo->Items.at(itemIdx); - Y_ABORT_UNLESS(item.State == EState::Transferring); + Y_ABORT_UNLESS(item.State == EState::Transferring || item.State == EState::CreateChangefeed); Y_ABORT_UNLESS(item.DstPathId); if (!Self->PathsById.contains(item.DstPathId)) { @@ -619,6 +619,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } switch (importInfo->Items.at(i).State) { + case EState::CreateChangefeed: case EState::Transferring: CancelTransferring(importInfo, i); break; @@ -752,10 +753,13 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase case EState::CreateSchemeObject: case EState::Transferring: case EState::BuildIndexes: + case EState::CreateChangefeed: if (item.WaitTxId == InvalidTxId) { if (!IsCreatedByQuery(item) || item.PreparedCreationQuery) { + Cerr << "AllocateTxIdCreateChfg: " << IsCreatedByQuery(item) << " " << item.PreparedCreationQuery << Endl; AllocateTxId(importInfo, itemIdx); } else { + Cerr << "NoAllocateTxIdCreateChfg SchemeQueryExecutor: " << IsCreatedByQuery(item) << " " << item.PreparedCreationQuery << Endl; const auto database = GetDatabase(*Self); item.SchemeQueryExecutor = ctx.Register(CreateSchemeQueryExecutor( Self->SelfId(), importInfo->Id, itemIdx, item.CreationQuery, database @@ -763,6 +767,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Self->RunningImportSchemeQueryExecutors.emplace(item.SchemeQueryExecutor); } } else { + Cerr << "SubscribeTxCreateChfg: " << IsCreatedByQuery(item) << " " << item.PreparedCreationQuery << Endl; SubscribeTx(importInfo, itemIdx); } break; @@ -777,6 +782,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase TTxId txId = InvalidTxId; switch (item.State) { + case EState::CreateChangefeed: case EState::Transferring: if (!CancelTransferring(importInfo, itemIdx)) { txId = GetActiveRestoreTxId(importInfo, itemIdx); @@ -798,6 +804,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Self->PersistImportItemState(db, importInfo, itemIdx); switch (item.State) { + case EState::CreateChangefeed: case EState::Transferring: CancelTransferring(importInfo, itemIdx); break; @@ -1061,7 +1068,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase )) { if (record.GetPathCreateTxId()) { txId = TTxId(record.GetPathCreateTxId()); - } else if (item.State == EState::Transferring) { + } else if (item.State == EState::Transferring || item.State == EState::CreateChangefeed) { txId = GetActiveRestoreTxId(importInfo, itemIdx); } } @@ -1076,7 +1083,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase item.WaitTxId = txId; Self->PersistImportItemState(db, importInfo, itemIdx); - if (importInfo->State != EState::Waiting && item.State == EState::Transferring) { + if (importInfo->State != EState::Waiting && (item.State == EState::Transferring || item.State == EState::CreateChangefeed)) { CancelTransferring(importInfo, itemIdx); return; } diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 3480af83dd56..6965a2ba288a 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5093,6 +5093,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE); + runtime.GetAppData().FeatureFlags.SetEnableChangefeedsImport(true); if (createsViews) { runtime.GetAppData().FeatureFlags.SetEnableViews(true); } @@ -5355,4 +5356,92 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { )" ); } + + Y_UNIT_TEST(ShouldSucceedOnSingleChangefeed) { + THashMap schemes; + + const auto changefeedName = "update_changefeed"; + + schemes.emplace("", R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )"); + + const auto changefeedDesc = Sprintf(R"( + name: "%s" + mode: MODE_UPDATES + format: FORMAT_JSON + state: STATE_ENABLED + )", changefeedName); + + const auto topicDesc = R"( + partitioning_settings { + min_active_partitions: 1 + max_active_partitions: 1 + auto_partitioning_settings { + strategy: AUTO_PARTITIONING_STRATEGY_DISABLED + partition_write_speed { + stabilization_window { + seconds: 300 + } + up_utilization_percent: 80 + down_utilization_percent: 20 + } + } + } + partitions { + active: true + } + retention_period { + seconds: 86400 + } + partition_write_speed_bytes_per_second: 1048576 + partition_write_burst_bytes: 1048576 + attributes { + key: "__max_partition_message_groups_seqno_stored" + value: "6000000" + } + attributes { + key: "_allow_unauthenticated_read" + value: "true" + } + attributes { + key: "_allow_unauthenticated_write" + value: "true" + } + attributes { + key: "_message_group_seqno_retention_period_ms" + value: "1382400000" + } + consumers { + name: "my_consumer" + read_from { + } + attributes { + key: "_service_type" + value: "data-streams" + } + } + )"; + + NAttr::TAttributes attr; + attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc); + + schemes.emplace("/update_feed", + TTypedScheme { + EPathTypeCdcStream, + changefeedDesc, + std::move(attr) + } + ); + + ShouldSucceed(schemes); + } } From d87b5f691ff1a860ed5ec8adf286c3db5bcf235b Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 18 Feb 2025 14:18:20 +0300 Subject: [PATCH 59/65] fix reboot --- .../schemeshard_import__create.cpp | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index ed53ff086622..0f5b256892d8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -522,6 +522,28 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Send(Self->SelfId(), CreateChangefeedPropose(Self, txId, item)); } + bool CancelCreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + if (item.WaitTxId == InvalidTxId) { + if (item.SubState == ESubState::Proposed) { + importInfo->State = EState::Cancellation; + } + + return false; + } + + importInfo->State = EState::Cancellation; + + LOG_I("TImport::TTxProgress: cancel restore's tx" + << ": info# " << importInfo->ToString() + << ", item# " << item.ToString(itemIdx)); + + Send(Self->SelfId(), CancelRestorePropose(importInfo, item.WaitTxId), 0, importInfo->Id); + return true; + } + void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); @@ -554,7 +576,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); const auto& item = importInfo->Items.at(itemIdx); - Y_ABORT_UNLESS(item.State == EState::Transferring || item.State == EState::CreateChangefeed); + Y_ABORT_UNLESS(item.State == EState::Transferring); Y_ABORT_UNLESS(item.DstPathId); if (!Self->PathsById.contains(item.DstPathId)) { @@ -584,6 +606,25 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase return TTxId(ui64((*infoPtr)->Id)); } + TTxId GetActiveCreateChangefeedTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { + Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); + const auto& item = importInfo->Items.at(itemIdx); + + Y_ABORT_UNLESS(item.State == EState::CreateChangefeed); + Y_ABORT_UNLESS(item.DstPathId); + + if (!Self->PathsById.contains(item.DstPathId)) { + return InvalidTxId; + } + + auto path = Self->PathsById.at(item.DstPathId); + if (path->PathState != NKikimrSchemeOp::EPathStateAlter) { + return InvalidTxId; + } + + return path->LastTxId; + } + static TString MakeIndexBuildUid(TImportInfo::TPtr importInfo, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); const auto& item = importInfo->Items.at(itemIdx); @@ -620,6 +661,8 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (importInfo->Items.at(i).State) { case EState::CreateChangefeed: + CancelCreateChangefeed(importInfo, i); + break; case EState::Transferring: CancelTransferring(importInfo, i); break; @@ -782,7 +825,12 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase TTxId txId = InvalidTxId; switch (item.State) { - case EState::CreateChangefeed: + case EState::CreateChangefeed: + if (!CancelCreateChangefeed(importInfo, itemIdx)) { + txId = GetActiveCreateChangefeedTxId(importInfo, itemIdx); + } + break; + case EState::Transferring: if (!CancelTransferring(importInfo, itemIdx)) { txId = GetActiveRestoreTxId(importInfo, itemIdx); @@ -805,6 +853,9 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (item.State) { case EState::CreateChangefeed: + CancelCreateChangefeed(importInfo, itemIdx); + break; + case EState::Transferring: CancelTransferring(importInfo, itemIdx); break; @@ -1068,8 +1119,10 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase )) { if (record.GetPathCreateTxId()) { txId = TTxId(record.GetPathCreateTxId()); - } else if (item.State == EState::Transferring || item.State == EState::CreateChangefeed) { + } else if (item.State == EState::Transferring) { txId = GetActiveRestoreTxId(importInfo, itemIdx); + } else if (item.State == EState::CreateChangefeed) { + txId = GetActiveCreateChangefeedTxId(importInfo, itemIdx); } } @@ -1083,11 +1136,16 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase item.WaitTxId = txId; Self->PersistImportItemState(db, importInfo, itemIdx); - if (importInfo->State != EState::Waiting && (item.State == EState::Transferring || item.State == EState::CreateChangefeed)) { + if (importInfo->State != EState::Waiting && item.State == EState::Transferring) { CancelTransferring(importInfo, itemIdx); return; } + if (importInfo->State != EState::Waiting && item.State == EState::CreateChangefeed) { + CancelCreateChangefeed(importInfo, itemIdx); + return; + } + if (item.State == EState::CreateSchemeObject) { auto createPath = TPath::Resolve(item.DstPathName, Self); Y_ABORT_UNLESS(createPath); From 52f83b733ecac082f377f59889b89039c5355414 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 18 Feb 2025 14:36:28 +0300 Subject: [PATCH 60/65] removed cancel create changefeed --- .../schemeshard_import__create.cpp | 38 +------------------ 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 0f5b256892d8..2122594b3241 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -522,28 +522,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Send(Self->SelfId(), CreateChangefeedPropose(Self, txId, item)); } - bool CancelCreateChangefeed(TImportInfo::TPtr importInfo, ui32 itemIdx) { - Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); - const auto& item = importInfo->Items.at(itemIdx); - - if (item.WaitTxId == InvalidTxId) { - if (item.SubState == ESubState::Proposed) { - importInfo->State = EState::Cancellation; - } - - return false; - } - - importInfo->State = EState::Cancellation; - - LOG_I("TImport::TTxProgress: cancel restore's tx" - << ": info# " << importInfo->ToString() - << ", item# " << item.ToString(itemIdx)); - - Send(Self->SelfId(), CancelRestorePropose(importInfo, item.WaitTxId), 0, importInfo->Id); - return true; - } - void AllocateTxId(TImportInfo::TPtr importInfo, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < importInfo->Items.size()); auto& item = importInfo->Items.at(itemIdx); @@ -660,9 +638,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase } switch (importInfo->Items.at(i).State) { - case EState::CreateChangefeed: - CancelCreateChangefeed(importInfo, i); - break; case EState::Transferring: CancelTransferring(importInfo, i); break; @@ -826,9 +801,7 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase switch (item.State) { case EState::CreateChangefeed: - if (!CancelCreateChangefeed(importInfo, itemIdx)) { - txId = GetActiveCreateChangefeedTxId(importInfo, itemIdx); - } + txId = GetActiveCreateChangefeedTxId(importInfo, itemIdx); break; case EState::Transferring: @@ -852,10 +825,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Self->PersistImportItemState(db, importInfo, itemIdx); switch (item.State) { - case EState::CreateChangefeed: - CancelCreateChangefeed(importInfo, itemIdx); - break; - case EState::Transferring: CancelTransferring(importInfo, itemIdx); break; @@ -1141,11 +1110,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase return; } - if (importInfo->State != EState::Waiting && item.State == EState::CreateChangefeed) { - CancelCreateChangefeed(importInfo, itemIdx); - return; - } - if (item.State == EState::CreateSchemeObject) { auto createPath = TPath::Resolve(item.DstPathName, Self); Y_ABORT_UNLESS(createPath); From e999a7e06bb4473060e05f632964830ec31256b8 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 18 Feb 2025 15:04:02 +0300 Subject: [PATCH 61/65] reboot cancel test --- ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 6965a2ba288a..47fb7ef30b68 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5357,7 +5357,7 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { ); } - Y_UNIT_TEST(ShouldSucceedOnSingleChangefeed) { + THashMap GetSchemeWithChangefeed() { THashMap schemes; const auto changefeedName = "update_changefeed"; @@ -5441,7 +5441,14 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { std::move(attr) } ); + return schemes; + } + + Y_UNIT_TEST(ShouldSucceedOnSingleChangefeed) { + ShouldSucceed(GetSchemeWithChangefeed()); + } - ShouldSucceed(schemes); + Y_UNIT_TEST(CancelShouldSucceedOnSingleChangefeed) { + CancelShouldSucceed(GetSchemeWithChangefeed()); } } From c448cc8d45737fcca16dc12aa3579281d6c84445 Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Tue, 18 Feb 2025 19:39:23 +0300 Subject: [PATCH 62/65] Apply suggestions from code review Co-authored-by: Ilnaz Nizametdinov --- .../ut_helpers/ut_backup_restore_common.h | 1 + ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp | 14 ++++---------- ydb/core/wrappers/ut_helpers/s3_mock.cpp | 3 +-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h b/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h index 1be58e907d96..07ba2c33aaf1 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ut_backup_restore_common.h @@ -23,6 +23,7 @@ namespace NAttr { enum class EKeys { TOPIC_DESCRIPTION, }; + class TAttributes : public THashMap { public: const TString& GetTopicDescription() const { diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 6df8b6a8eb72..5ed918282cdf 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -4825,7 +4825,7 @@ Y_UNIT_TEST_SUITE(TImportTests) { }); } - struct GeneratedChangefeed { + struct TGeneratedChangefeed { std::pair Changefeed; std::function Checker; }; @@ -4894,14 +4894,8 @@ Y_UNIT_TEST_SUITE(TImportTests) { NAttr::TAttributes attr; attr.emplace(NAttr::EKeys::TOPIC_DESCRIPTION, topicDesc); return { - {changefeedPath, GenerateTestData( - { - EPathTypeCdcStream, - changefeedDesc, - std::move(attr) - } - )}, - [changefeedPath = TString(changefeedPath)](TTestBasicRuntime& runtime){ + {changefeedPath, GenerateTestData({EPathTypeCdcStream, changefeedDesc, std::move(attr)})}, + [changefeedPath = TString(changefeedPath)](TTestBasicRuntime& runtime) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table" + changefeedPath, false, false, true), { NLs::PathExist }); @@ -4934,7 +4928,7 @@ Y_UNIT_TEST_SUITE(TImportTests) { )", {{"a", 1}}, permissions); bucketContent.emplace("", data); - return [](TTestBasicRuntime& runtime){ + return [](TTestBasicRuntime& runtime) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { NLs::PathExist }); diff --git a/ydb/core/wrappers/ut_helpers/s3_mock.cpp b/ydb/core/wrappers/ut_helpers/s3_mock.cpp index 0a620fdc2714..d131967e4ea1 100644 --- a/ydb/core/wrappers/ut_helpers/s3_mock.cpp +++ b/ydb/core/wrappers/ut_helpers/s3_mock.cpp @@ -170,7 +170,6 @@ TString BuildContentListXML(const TVector& paths) { } TString BuildListObjectsXML(const TVector& paths, const TStringBuf bucketName) { - return Sprintf(R"( @@ -184,9 +183,9 @@ TString BuildListObjectsXML(const TVector& paths, const TStringBuf buck bool TS3Mock::TRequest::HttpServeList(const TReplyParams& params, TStringBuf bucketName, const TString& prefix) { Cerr << "S3_MOCK::HttpServeList: " << prefix << Endl; params.Output << "HTTP/1.1 200 Ok\r\n"; - TVector paths; THttpHeaders headers; + TVector paths; for (const auto& [key, value] : Parent->Data) { TFsPath path = key; if (path.IsSubpathOf(TStringBuilder() << bucketName << "/" << prefix)) { From 9f7a019b3a7deaf6486cbcbeabd1449f63f5036d Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 18 Feb 2025 19:54:05 +0300 Subject: [PATCH 63/65] review --- contrib/python/ydb/py2/ydb/import_client.py | 5 +---- ydb/core/tx/schemeshard/schemeshard_import__create.cpp | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/contrib/python/ydb/py2/ydb/import_client.py b/contrib/python/ydb/py2/ydb/import_client.py index 851aa392d366..a11d77a0c688 100644 --- a/contrib/python/ydb/py2/ydb/import_client.py +++ b/contrib/python/ydb/py2/ydb/import_client.py @@ -31,10 +31,7 @@ class ImportProgress(enum.IntEnum): def _initialize_progresses(): for key, value in ydb_import_pb2.ImportProgress.Progress.items(): - try: - _progresses[value] = getattr(ImportProgress, key[len("PROGRESS_") :]) - except AttributeError: - pass + _progresses[value] = getattr(ImportProgress, key[len("PROGRESS_") :]) _initialize_progresses() diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 3cf4adbc9cdd..e21149edb9f8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -793,10 +793,8 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase case EState::CreateChangefeed: if (item.WaitTxId == InvalidTxId) { if (!IsCreatedByQuery(item) || item.PreparedCreationQuery) { - Cerr << "AllocateTxIdCreateChfg: " << IsCreatedByQuery(item) << " " << item.PreparedCreationQuery << Endl; AllocateTxId(importInfo, itemIdx); } else { - Cerr << "NoAllocateTxIdCreateChfg SchemeQueryExecutor: " << IsCreatedByQuery(item) << " " << item.PreparedCreationQuery << Endl; const auto database = GetDatabase(*Self); item.SchemeQueryExecutor = ctx.Register(CreateSchemeQueryExecutor( Self->SelfId(), importInfo->Id, itemIdx, item.CreationQuery, database @@ -804,7 +802,6 @@ struct TSchemeShard::TImport::TTxProgress: public TSchemeShard::TXxport::TTxBase Self->RunningImportSchemeQueryExecutors.emplace(item.SchemeQueryExecutor); } } else { - Cerr << "SubscribeTxCreateChfg: " << IsCreatedByQuery(item) << " " << item.PreparedCreationQuery << Endl; SubscribeTx(importInfo, itemIdx); } break; From 54d021f885cbbb36984830eedd738d57d769bd7b Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 18 Feb 2025 20:12:44 +0300 Subject: [PATCH 64/65] review tests --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 80 ++++++++----------- 1 file changed, 32 insertions(+), 48 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 5ed918282cdf..42db33d48888 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -5016,36 +5016,20 @@ Y_UNIT_TEST_SUITE(TImportTests) { } } - void TestImportChangefeed() { - TestImportChangefeeds(1, AddedScheme); - } - - void TestImportChangefeeds() { - TestImportChangefeeds(3, AddedScheme); - } - - void TestImportChangefeedWithTablePermissions() { - TestImportChangefeeds(1, AddedSchemeWithPermissions); - } - - void TestImportChangefeedsWithTablePermissions() { - TestImportChangefeeds(3, AddedSchemeWithPermissions); - } - Y_UNIT_TEST(Changefeed) { - TestImportChangefeed(); + TestImportChangefeeds(1, AddedScheme); } Y_UNIT_TEST(Changefeeds) { - TestImportChangefeeds(); + TestImportChangefeeds(3, AddedScheme); } Y_UNIT_TEST(ChangefeedWithTablePermissions) { - TestImportChangefeedWithTablePermissions(); + TestImportChangefeeds(1, AddedSchemeWithPermissions); } Y_UNIT_TEST(ChangefeedsWithTablePermissions) { - TestImportChangefeedsWithTablePermissions(); + TestImportChangefeeds(3, AddedSchemeWithPermissions); } } @@ -5416,51 +5400,51 @@ Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { const auto topicDesc = R"( partitioning_settings { - min_active_partitions: 1 - max_active_partitions: 1 - auto_partitioning_settings { - strategy: AUTO_PARTITIONING_STRATEGY_DISABLED - partition_write_speed { - stabilization_window { - seconds: 300 - } - up_utilization_percent: 80 - down_utilization_percent: 20 + min_active_partitions: 1 + max_active_partitions: 1 + auto_partitioning_settings { + strategy: AUTO_PARTITIONING_STRATEGY_DISABLED + partition_write_speed { + stabilization_window { + seconds: 300 + } + up_utilization_percent: 80 + down_utilization_percent: 20 + } } } - } partitions { - active: true + active: true } retention_period { - seconds: 86400 + seconds: 86400 } partition_write_speed_bytes_per_second: 1048576 partition_write_burst_bytes: 1048576 attributes { - key: "__max_partition_message_groups_seqno_stored" - value: "6000000" + key: "__max_partition_message_groups_seqno_stored" + value: "6000000" } attributes { - key: "_allow_unauthenticated_read" - value: "true" + key: "_allow_unauthenticated_read" + value: "true" } attributes { - key: "_allow_unauthenticated_write" - value: "true" + key: "_allow_unauthenticated_write" + value: "true" } attributes { - key: "_message_group_seqno_retention_period_ms" - value: "1382400000" + key: "_message_group_seqno_retention_period_ms" + value: "1382400000" } consumers { - name: "my_consumer" - read_from { - } - attributes { - key: "_service_type" - value: "data-streams" - } + name: "my_consumer" + read_from { + } + attributes { + key: "_service_type" + value: "data-streams" + } } )"; From f3ca9e4084225d5c6df5d10c270d669724bb5cd2 Mon Sep 17 00:00:00 2001 From: st-shchetinin Date: Tue, 18 Feb 2025 20:48:05 +0300 Subject: [PATCH 65/65] ut_restore fix --- .../tx/schemeshard/ut_restore/ut_restore.cpp | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 42db33d48888..ec6cb468cb45 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -4830,7 +4830,7 @@ Y_UNIT_TEST_SUITE(TImportTests) { std::function Checker; }; - GeneratedChangefeed GenChangefeed(ui64 num = 1) { + TGeneratedChangefeed GenChangefeed(ui64 num = 1) { const TString changefeedName = TStringBuilder() << "updates_feed" << num; const auto changefeedPath = TStringBuilder() << "/" << changefeedName; @@ -4843,51 +4843,51 @@ Y_UNIT_TEST_SUITE(TImportTests) { const auto topicDesc = R"( partitioning_settings { - min_active_partitions: 1 - max_active_partitions: 1 - auto_partitioning_settings { - strategy: AUTO_PARTITIONING_STRATEGY_DISABLED - partition_write_speed { - stabilization_window { - seconds: 300 - } - up_utilization_percent: 80 - down_utilization_percent: 20 + min_active_partitions: 1 + max_active_partitions: 1 + auto_partitioning_settings { + strategy: AUTO_PARTITIONING_STRATEGY_DISABLED + partition_write_speed { + stabilization_window { + seconds: 300 + } + up_utilization_percent: 80 + down_utilization_percent: 20 + } } } - } partitions { - active: true + active: true } retention_period { - seconds: 86400 + seconds: 86400 } partition_write_speed_bytes_per_second: 1048576 partition_write_burst_bytes: 1048576 attributes { - key: "__max_partition_message_groups_seqno_stored" - value: "6000000" + key: "__max_partition_message_groups_seqno_stored" + value: "6000000" } attributes { - key: "_allow_unauthenticated_read" - value: "true" + key: "_allow_unauthenticated_read" + value: "true" } attributes { - key: "_allow_unauthenticated_write" - value: "true" + key: "_allow_unauthenticated_write" + value: "true" } attributes { - key: "_message_group_seqno_retention_period_ms" - value: "1382400000" + key: "_message_group_seqno_retention_period_ms" + value: "1382400000" } consumers { - name: "my_consumer" - read_from { - } - attributes { - key: "_service_type" - value: "data-streams" - } + name: "my_consumer" + read_from { + } + attributes { + key: "_service_type" + value: "data-streams" + } } )";