Sparsed columns are disabled by default and can be optionally enabled (#9002)

Author: aavdonkin

ydb/core/kqp/ut/common/kqp_ut_common.h

@@ -97,6 +97,7 @@ struct TKikimrSettings: public TTestFeatureFlagsHolder<TKikimrSettings> {
         exchangerSettings->SetStartDelayMs(10);
         exchangerSettings->SetMaxDelayMs(10);
         AppConfig.MutableColumnShardConfig()->SetDisabledOnSchemeShard(false);
+        FeatureFlags.SetEnableSparsedColumns(true);
     }
 
     TKikimrSettings& SetAppConfig(const NKikimrConfig::TAppConfig& value) { AppConfig = value; return *this; }

ydb/core/protos/feature_flags.proto

@@ -160,4 +160,5 @@ message TFeatureFlags {
     optional bool EnableMetadataObjectsOnServerless = 141 [default = true];
     optional bool EnableOlapCompression = 142 [default = false];
     optional bool EnableExternalDataSourcesOnServerless = 143 [default = true];
+    optional bool EnableSparsedColumns = 144 [default = false];
 }

ydb/core/tx/schemeshard/olap/operations/alter/common/update.h

@@ -2,6 +2,7 @@
 #include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
 #include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h>
 #include <ydb/core/tx/schemeshard/olap/table/table.h>
+#include <ydb/core/formats/arrow/accessor/common/const.h>
 
 namespace NKikimr::NSchemeShard::NOlap::NAlter {
 
@@ -65,6 +66,17 @@ class TColumnTableUpdate: public ISSEntityUpdate {
         return result;
     }
 
+    bool CheckTargetSchema(const TOlapSchema& targetSchema) {
+        if (!AppData()->FeatureFlags.GetEnableSparsedColumns()) {
+            for (auto& [_, column]: targetSchema.GetColumns().GetColumns()) {
+                if (column.GetDefaultValue().GetValue() || (column.GetAccessorConstructor().GetClassName() == NKikimr::NArrow::NAccessor::TGlobalConst::SparsedDataAccessorName)) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
 public:
 };
 

ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp

@@ -1,6 +1,7 @@
 #include "update.h"
 #include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h>
 #include <ydb/core/tx/schemeshard/olap/common/common.h>
+#include <ydb/core/formats/arrow/accessor/common/const.h>
 
 namespace NKikimr::NSchemeShard::NOlap::NAlter {
 
@@ -51,6 +52,9 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitializeImpl(const TUpda
         }
     }
 
+    if (!CheckTargetSchema(targetSchema)) {
+        return TConclusionStatus::Fail("schema update error: sparsed columns are disabled");
+    }
     auto description = originalTable.GetTableInfoVerified().Description;
     targetSchema.Serialize(*description.MutableSchema());
     auto ttl = originalTable.GetTableTTLOptional() ? *originalTable.GetTableTTLOptional() : TOlapTTL();

ydb/core/tx/schemeshard/olap/operations/alter_store.cpp

@@ -1,6 +1,7 @@
 #include <ydb/core/tx/schemeshard/schemeshard__operation_part.h>
 #include <ydb/core/tx/schemeshard/schemeshard__operation_common.h>
 #include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+#include <ydb/core/formats/arrow/accessor/common/const.h>
 
 #include "checks.h"
 
@@ -525,6 +526,17 @@ class TAlterOlapStore: public TSubOperation {
             return result;
         }
 
+        if (!AppData()->FeatureFlags.GetEnableSparsedColumns()) {
+            for (auto& [_, preset]: alterData->SchemaPresets) {
+                for (auto& [_, column]: preset.GetColumns().GetColumns()) {
+                    if (column.GetDefaultValue().GetValue() || (column.GetAccessorConstructor().GetClassName() == NKikimr::NArrow::NAccessor::TGlobalConst::SparsedDataAccessorName)) {
+                        result->SetError(NKikimrScheme::StatusSchemeError,"schema update error: sparsed columns are disabled");
+                        return result;
+                    }
+                }
+            }
+        }
+
         auto domainInfo = parentPath.DomainInfo();
         const TSchemeLimits& limits = domainInfo->GetSchemeLimits();
 

ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp

@@ -557,6 +557,17 @@ Y_UNIT_TEST_SUITE(TOlap) {
                 }
             }
         )", {NKikimrScheme::StatusAccepted});
+
+        env.TestWaitNotification(runtime, txId);
+        TestAlterOlapStore(runtime, ++txId, "/MyRoot", R"(
+            Name: "OlapStore"
+            AlterSchemaPresets {
+                Name: "default"
+                AlterSchema {
+                    AlterColumns { Name: "comment" DefaultValue: "10" }
+                }
+            }
+        )", {NKikimrScheme::StatusSchemeError});
     }
 
     Y_UNIT_TEST(AlterTtl) {

ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp

@@ -1150,6 +1150,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) {
                 Columns { Name: "key" Type: "Uint64" NotNull: true }
                 Columns { Name: "modified_at" Type: "Timestamp" }
                 Columns { Name: "saved_at" Type: "Datetime" }
+                Columns { Name: "data" Type: "Utf8" }
                 KeyColumnNames: ["key"]
             }
         )");
@@ -1206,6 +1207,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) {
                 }
             }
         );
+        TestAlterColumnTable(runtime, ++txId, "/MyRoot", R"(
+            Name: "TTLEnabledTable"
+            AlterSchema {
+                AlterColumns {Name: "data" DefaultValue: "10"}
+            }
+        )", {NKikimrScheme::StatusSchemeError});
+        env.TestWaitNotification(runtime, txId);
     }
 
     Y_UNIT_TEST(AlterColumnTable_Negative) {