Added another cost based optimization level (#8348)

Author: pavelvelikhov

ydb/core/kqp/opt/kqp_statistics_transformer.cpp

@@ -79,7 +79,8 @@ void InferStatisticsForReadTable(const TExprNode::TPtr& input, TTypeAnnotationCo
         byteSize, 
         0.0, 
         keyColumns,
-        inputStats->ColumnStatistics);
+        inputStats->ColumnStatistics,
+        inputStats->StorageType);
 
     YQL_CLOG(TRACE, CoreDq) << "Infer statistics for read table" << stats->ToString();
 
@@ -119,6 +120,19 @@ void InferStatisticsForKqpTable(const TExprNode::TPtr& input, TTypeAnnotationCon
         }
     }
 
+    EStorageType storageType = EStorageType::NA;
+    switch (tableData.Metadata->Kind) {
+        case EKikimrTableKind::Datashard:
+            storageType = EStorageType::RowStorage;
+            break;
+        case EKikimrTableKind::Olap:
+            storageType = EStorageType::ColumnStorage;
+            break;
+        default:
+            break;
+    }
+    stats->StorageType = storageType;
+
     YQL_CLOG(TRACE, CoreDq) << "Infer statistics for table: " << path.Value() << ": " << stats->ToString();
 
     typeCtx->SetStats(input.Get(), stats);
@@ -151,7 +165,8 @@ void InferStatisticsForSteamLookup(const TExprNode::TPtr& input, TTypeAnnotation
         byteSize, 
         0, 
         inputStats->KeyColumns,
-        inputStats->ColumnStatistics);
+        inputStats->ColumnStatistics,
+        inputStats->StorageType);
 
     typeCtx->SetStats(input.Get(), res); 
 
@@ -195,7 +210,8 @@ void InferStatisticsForLookupTable(const TExprNode::TPtr& input, TTypeAnnotation
         byteSize, 
         0, 
         inputStats->KeyColumns,
-        inputStats->ColumnStatistics));
+        inputStats->ColumnStatistics,
+        inputStats->StorageType));
 }
 
 /**
@@ -250,7 +266,8 @@ void InferStatisticsForRowsSourceSettings(const TExprNode::TPtr& input, TTypeAnn
         byteSize, 
         cost, 
         keyColumns, 
-        inputStats->ColumnStatistics));
+        inputStats->ColumnStatistics,
+        inputStats->StorageType));
 }
 
 /**
@@ -282,7 +299,8 @@ void InferStatisticsForReadTableIndexRanges(const TExprNode::TPtr& input, TTypeA
         inputStats->ByteSize, 
         inputStats->Cost, 
         indexColumnsPtr,
-        inputStats->ColumnStatistics);
+        inputStats->ColumnStatistics,
+        inputStats->StorageType);
 
     typeCtx->SetStats(input.Get(), stats);
 

ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp

@@ -148,7 +148,7 @@ bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptr<IBaseOptimizerN
 
     switch( joinAlgo ) {
         case EJoinAlgoType::LookupJoin:
-            if ((OptLevel >= 2) && (left->Stats->Nrows > 1000)) {
+            if ((OptLevel != 3) && (left->Stats->Nrows > 1000)) {
                 return false;
             }
             return IsLookupJoinApplicable(left, right, joinConditions, leftJoinKeys, rightJoinKeys, *this);
@@ -157,7 +157,7 @@ bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptr<IBaseOptimizerN
             if (joinKind != EJoinKind::LeftSemi) {
                 return false;
             }
-            if ((OptLevel >= 2) && (right->Stats->Nrows > 1000)) {
+            if ((OptLevel != 3) && (right->Stats->Nrows > 1000)) {
                 return false;
             }
             return IsLookupJoinApplicable(right, left, joinConditions, rightJoinKeys, leftJoinKeys, *this);
@@ -176,13 +176,13 @@ double TKqpProviderContext::ComputeJoinCost(const TOptimizerStatistics& leftStat
 
     switch(joinAlgo) {
         case EJoinAlgoType::LookupJoin:
-            if (OptLevel == 2) {
+            if (OptLevel == 3) {
                 return -1;
             }
             return leftStats.Nrows + outputRows;
 
         case EJoinAlgoType::LookupJoinReverse:
-            if (OptLevel == 2) {
+            if (OptLevel == 3) {
                 return -1;
             }
             return rightStats.Nrows + outputRows;

ydb/core/kqp/opt/logical/kqp_opt_log.cpp

@@ -166,14 +166,14 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
     }
 
     TMaybeNode<TExprBase> RewriteEquiJoin(TExprBase node, TExprContext& ctx) {
-        bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) == 3;
+        bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) >= 2;
         TExprBase output = DqRewriteEquiJoin(node, KqpCtx.Config->GetHashJoinMode(), useCBO, ctx, TypesCtx, KqpCtx.JoinsCount);
         DumpAppliedRule("RewriteEquiJoin", node.Ptr(), output.Ptr(), ctx);
         return output;
     }
 
     TMaybeNode<TExprBase> JoinToIndexLookup(TExprBase node, TExprContext& ctx) {
-        bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) == 3;
+        bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) >= 2;
         TExprBase output = KqpJoinToIndexLookup(node, ctx, KqpCtx, useCBO);
         DumpAppliedRule("JoinToIndexLookup", node.Ptr(), output.Ptr(), ctx);
         return output;

ydb/core/kqp/provider/yql_kikimr_settings.h

@@ -169,7 +169,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi
     bool EnableOltpSink = false;
     NKikimrConfig::TTableServiceConfig_EBlockChannelsMode BlockChannelsMode;
     bool EnableSpillingGenericQuery = false;
-    ui32 DefaultCostBasedOptimizationLevel = 3;
+    ui32 DefaultCostBasedOptimizationLevel = 4;
     bool EnableConstantFolding = true;
     ui64 DefaultEnableSpillingNodes = 0;