Outline

be/src/exec/CMakeLists.txt

@@ -26,6 +26,8 @@ file(GLOB_RECURSE EXEC_FILES CONFIGURE_DEPENDS *.cpp)
 if (WITH_LZO)
     set(EXEC_FILES ${EXEC_FILES}
         lzo_decompressor.cpp
+            schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.cpp
+            schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h
     )
 endif()
 

be/src/exec/schema_scanner.cpp

@@ -62,6 +62,7 @@
 #include "runtime/define_primitive_type.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/types.h"
+#include "schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h"
 #include "util/string_util.h"
 #include "util/types.h"
 #include "vec/columns/column.h"
@@ -231,6 +232,8 @@ std::unique_ptr<SchemaScanner> SchemaScanner::create(TSchemaTableType::type type
         return SchemaBackendKerberosTicketCacheScanner::create_unique();
     case TSchemaTableType::SCH_ROUTINE_LOAD_JOBS:
         return SchemaRoutineLoadJobScanner::create_unique();
+    case TSchemaTableType::SCH_OPTIMIZER_SQL_PLAN_OUTLINE:
+        return SchemaOptimizerSqlPlanOutlineScanner::create_unique();
     default:
         return SchemaDummyScanner::create_unique();
         break;

be/src/exec/schema_scanner/schema_helper.cpp

@@ -151,4 +151,13 @@ Status SchemaHelper::fetch_routine_load_job(const std::string& ip, const int32_t
             });
 }
 
+Status SchemaHelper::fetch_outline_info(const std::string& ip, const int32_t port,
+                                            const TFetchOutlineInfoRequest& request,
+                                            TFetchOutlineInfoResult* result) {
+    return ThriftRpcHelper::rpc<FrontendServiceClient>(
+            ip, port, [&request, &result](FrontendServiceConnection& client) {
+                client->fetchOutlineInfo(*result, request);
+            });
+}
+
 } // namespace doris

be/src/exec/schema_scanner/schema_helper.h

@@ -26,6 +26,8 @@
 namespace doris {
 class TDescribeTablesParams;
 class TDescribeTablesResult;
+class TFetchOutlineInfoRequest;
+class TFetchOutlineInfoResult;
 class TFetchRoutineLoadJobRequest;
 class TFetchRoutineLoadJobResult;
 class TGetDbsParams;
@@ -90,6 +92,10 @@ class SchemaHelper {
     static Status fetch_routine_load_job(const std::string& ip, const int32_t port,
                                          const TFetchRoutineLoadJobRequest& request,
                                          TFetchRoutineLoadJobResult* result);
+
+    static Status fetch_outline_info(const std::string& ip, const int32_t port,
+                                         const TFetchOutlineInfoRequest& request,
+                                         TFetchOutlineInfoResult* result);
 };
 
 } // namespace doris

be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.cpp

@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h"
+
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/FrontendService_types.h>
+
+#include <string>
+
+#include "exec/schema_scanner/schema_helper.h"
+#include "runtime/runtime_state.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/block.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+namespace doris {
+class RuntimeState;
+namespace vectorized {
+class Block;
+} // namespace vectorized
+
+std::vector<SchemaScanner::ColumnDesc> SchemaOptimizerSqlPlanOutlineScanner::_s_tbls_columns = {
+        //   name,       type,          size,     is_null
+        {"OUTLINE_NAME", TYPE_STRING, sizeof(StringRef), true},
+        {"VISIBLE_SIGNATURE", TYPE_STRING, sizeof(StringRef), true},
+        {"SQL_ID", TYPE_STRING, sizeof(StringRef), true},
+        {"SQL_TEXT", TYPE_STRING, sizeof(StringRef), true},
+        {"OUTLINE_TARGET", TYPE_STRING, sizeof(StringRef), true},
+        {"OUTLINE_DATA", TYPE_STRING, sizeof(StringRef), true},
+};
+
+SchemaOptimizerSqlPlanOutlineScanner::SchemaOptimizerSqlPlanOutlineScanner()
+        : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_OPTIMIZER_SQL_PLAN_OUTLINE) {}
+
+SchemaOptimizerSqlPlanOutlineScanner::~SchemaOptimizerSqlPlanOutlineScanner() {}
+
+Status SchemaOptimizerSqlPlanOutlineScanner::start(RuntimeState* state) {
+    if (!_is_init) {
+        return Status::InternalError("used before initialized.");
+    }
+    TFetchOutlineInfoRequest request;
+    RETURN_IF_ERROR(SchemaHelper::fetch_outline_info(
+            *(_param->common_param->ip), _param->common_param->port, request, &_result));
+    return Status::OK();
+}
+
+Status SchemaOptimizerSqlPlanOutlineScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
+    if (!_is_init) {
+        return Status::InternalError("call this before initial.");
+    }
+    if (block == nullptr || eos == nullptr) {
+        return Status::InternalError("invalid parameter.");
+    }
+
+    *eos = true;
+    if (_result.outlineInfos.empty()) {
+        return Status::OK();
+    }
+
+    return _fill_block_impl(block);
+}
+
+Status SchemaOptimizerSqlPlanOutlineScanner::_fill_block_impl(vectorized::Block* block) {
+    SCOPED_TIMER(_fill_block_timer);
+
+    const auto& outline_infos = _result.outlineInfos;
+    size_t row_num = outline_infos.size();
+    if (row_num == 0) {
+        return Status::OK();
+    }
+
+    for (size_t col_idx = 0; col_idx < _s_tbls_columns.size(); ++col_idx) {
+        const auto& col_desc = _s_tbls_columns[col_idx];
+
+        std::vector<StringRef> str_refs(row_num);
+        std::vector<int32_t> int_vals(row_num);
+        std::vector<int8_t> bool_vals(row_num);
+        std::vector<void*> datas(row_num);
+        std::vector<std::string> column_values(row_num);
+
+        for (size_t row_idx = 0; row_idx < row_num; ++row_idx) {
+            const auto& outline_info = outline_infos[row_idx];
+            std::string& column_value = column_values[row_idx];
+
+            if (col_desc.type == TYPE_STRING) {
+                switch (col_idx) {
+                case 0: // OUTLINE_NAME
+                    column_value = outline_info.__isset.outline_name ? outline_info.outline_name : "";
+                    break;
+                case 1: // VISIBLE_SIGNATURE
+                    column_value = outline_info.__isset.visible_signature ? outline_info.visible_signature : "";
+                    break;
+                case 2: // SQL_ID
+                    column_value = outline_info.__isset.sql_id ? outline_info.sql_id : "";
+                    break;
+                case 3: // SQL_TEXT
+                    column_value = outline_info.__isset.sql_text ? outline_info.sql_text : "";
+                    break;
+                case 4: // OUTLINE_TARGET
+                    column_value = outline_info.__isset.outline_target ? outline_info.outline_target : "";
+                    break;
+                case 5: // OUTLINE_DATA
+                    column_value = outline_info.__isset.outline_data ? outline_info.outline_data : "";
+                    break;
+                }
+
+                str_refs[row_idx] =
+                        StringRef(column_values[row_idx].data(), column_values[row_idx].size());
+                datas[row_idx] = &str_refs[row_idx];
+            }
+        }
+
+        RETURN_IF_ERROR(fill_dest_column_for_range(block, col_idx, datas));
+    }
+
+    return Status::OK();
+}
+
+} // namespace doris

be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h

@@ -0,0 +1,52 @@
+//SCHEMA_SCH_OPTIMIZER_SQL_PLAN_OUTLINE_SCANNER_H
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/FrontendService_types.h>
+
+#include <vector>
+
+#include "common/status.h"
+#include "exec/schema_scanner.h"
+
+namespace doris {
+    class RuntimeState;
+    namespace vectorized {
+        class Block;
+    } // namespace vectorized
+
+    class SchemaOptimizerSqlPlanOutlineScanner : public SchemaScanner {
+        ENABLE_FACTORY_CREATOR(SchemaOptimizerSqlPlanOutlineScanner);
+
+    public:
+        SchemaOptimizerSqlPlanOutlineScanner();
+        ~SchemaOptimizerSqlPlanOutlineScanner() override;
+
+        Status start(RuntimeState* state) override;
+        Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
+
+    private:
+        Status _fill_block_impl(vectorized::Block* block);
+
+        TFetchOutlineInfoResult _result;
+        static std::vector<SchemaScanner::ColumnDesc> _s_tbls_columns;
+    };
+
+} // namespace doris

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4

@@ -380,6 +380,7 @@ OR: 'OR';
 ORDER: 'ORDER';
 OUTER: 'OUTER';
 OUTFILE: 'OUTFILE';
+OUTLINE: 'OUTLINE';
 OVER: 'OVER';
 OVERWRITE: 'OVERWRITE';
 PARAMETER: 'PARAMETER';

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4

@@ -68,6 +68,7 @@ statementBase
     | supportedKillStatement            #supportedKillStatementAlias
     | supportedStatsStatement           #supportedStatsStatementAlias
     | supportedTransactionStatement     #supportedTransactionStatementAlias
+    | supportedOutlineStatement         #supportedOutlineStatementAlias
     | unsupportedStatement              #unsupported
     ;
 
@@ -374,6 +375,12 @@ supportedLoadStatement
     | createRoutineLoad                                                             #createRoutineLoadAlias
     ;
 
+supportedOutlineStatement
+    : CREATE (OR REPLACE)? OUTLINE outline_name=identifierOrText ON (query (TO query)? |
+      sql_id=STRING_LITERAL USING HINT_START identifier HINT_END )                  #createOutline
+    | DROP OUTLINE (IF EXISTS)? outline_name=identifierOrText                       #dropOutline
+    ;
+
 supportedOtherStatement
     : HELP mark=identifierOrText                                                    #help
     | UNLOCK TABLES                                                                 #unlockTables
@@ -1980,6 +1987,7 @@ nonReserved
     | ONLY
     | OPEN
     | OPTIMIZED
+    | OUTLINE
     | PARAMETER
     | PARSED
     | PASSWORD

fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java

@@ -91,8 +91,10 @@ public enum SchemaTableType {
             TSchemaTableType.SCH_CATALOG_META_CACHE_STATISTICS),
     SCH_BACKEND_KERBEROS_TICKET_CACHE("BACKEND_KERBEROS_TICKET_CACHE", "BACKEND_KERBEROS_TICKET_CACHE",
             TSchemaTableType.SCH_BACKEND_KERBEROS_TICKET_CACHE),
-    SCH_ROUTINE_LOAD_JOBS("ROUTINE_LOAD_JOBS", "ROUTINE_LOAD_JOBS",
-            TSchemaTableType.SCH_ROUTINE_LOAD_JOBS);
+    SCH_ROUTINE_LOAD_JOB("ROUTINE_LOAD_JOB", "ROUTINE_LOAD_JOB",
+            TSchemaTableType.SCH_ROUTINE_LOAD_JOBS),
+    SCH_OPTIMIZER_SQL_PLAN_OUTLINE("OPTIMIZER_SQL_PLAN_OUTLINE", "OPTIMIZER_SQL_PLAN_OUTLINE",
+                                      TSchemaTableType.SCH_OPTIMIZER_SQL_PLAN_OUTLINE);
 
     private static final String dbName = "INFORMATION_SCHEMA";
     private static SelectList fullSelectLists;

fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java

@@ -200,6 +200,7 @@
 import org.apache.doris.mysql.privilege.AccessControllerManager;
 import org.apache.doris.mysql.privilege.Auth;
 import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.hint.OutlineMgr;
 import org.apache.doris.nereids.jobs.load.LabelProcessor;
 import org.apache.doris.nereids.stats.HboPlanStatisticsManager;
 import org.apache.doris.nereids.trees.plans.commands.AlterSystemCommand;
@@ -391,6 +392,7 @@ public class Env {
     private StreamLoadRecordMgr streamLoadRecordMgr;
     private TabletLoadIndexRecorderMgr tabletLoadIndexRecorderMgr;
     private RoutineLoadManager routineLoadManager;
+    private OutlineMgr outlineMgr;
     private GroupCommitManager groupCommitManager;
     private SqlBlockRuleMgr sqlBlockRuleMgr;
     private ExportMgr exportMgr;
@@ -713,6 +715,7 @@ public Env(boolean isCheckpointCatalog) {
         this.catalogMgr = new CatalogMgr();
         this.load = new Load();
         this.routineLoadManager = EnvFactory.getInstance().createRoutineLoadManager();
+        this.outlineMgr = new OutlineMgr();
         this.groupCommitManager = new GroupCommitManager();
         this.sqlBlockRuleMgr = new SqlBlockRuleMgr();
         this.exportMgr = new ExportMgr();
@@ -2403,6 +2406,13 @@ public long loadGlobalVariable(DataInputStream in, long checksum) throws IOExcep
         return checksum;
     }
 
+    // outline persistence
+    public long loadOutline(DataInputStream in, long checksum) throws IOException, DdlException {
+        VariableMgr.read(in);
+        LOG.info("finished replay globalVariable from image");
+        return checksum;
+    }
+
     // load binlogs
     public long loadBinlogs(DataInputStream dis, long checksum) throws IOException {
         binlogManager.read(dis, checksum);
@@ -2422,6 +2432,12 @@ public long loadRoutineLoadJobs(DataInputStream dis, long checksum) throws IOExc
         return checksum;
     }
 
+    public long loadOutlines(DataInputStream dis, long checksum) throws IOException {
+        Env.getCurrentEnv().getOutlineMgr().readFields(dis);
+        LOG.info("finished replay outlines from image");
+        return checksum;
+    }
+
     public long loadLoadJobsV2(DataInputStream in, long checksum) throws IOException {
         loadManager.readFields(in);
         LOG.info("finished replay loadJobsV2 from image");
@@ -2730,6 +2746,11 @@ public long saveRoutineLoadJobs(CountingDataOutputStream dos, long checksum) thr
         return checksum;
     }
 
+    public long saveOutlines(CountingDataOutputStream dos, long checksum) throws IOException {
+        Env.getCurrentEnv().getOutlineMgr().write(dos);
+        return checksum;
+    }
+
     public long saveGlobalVariable(CountingDataOutputStream dos, long checksum) throws IOException {
         VariableMgr.write(dos);
         return checksum;
@@ -4658,6 +4679,10 @@ public RoutineLoadManager getRoutineLoadManager() {
         return routineLoadManager;
     }
 
+    public OutlineMgr getOutlineMgr() {
+        return outlineMgr;
+    }
+
     public GroupCommitManager getGroupCommitManager() {
         return groupCommitManager;
     }

fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java

@@ -620,7 +620,14 @@ public class SchemaTable extends Table {
                                     .column("CURRENT_ABORT_TASK_NUM", ScalarType.createType(PrimitiveType.INT))
                                     .column("IS_ABNORMAL_PAUSE", ScalarType.createType(PrimitiveType.BOOLEAN))
                                     .build())
-            )
+            ).put("optimizer_sql_plan_outline",
+                new SchemaTable(SystemIdGenerator.getNextId(), "optimizer_sql_plan_outline", TableType.SCHEMA,
+                    builder().column("OUTLINE_NAME", ScalarType.createStringType())
+                        .column("VISIBLE_SIGNATURE", ScalarType.createStringType())
+                        .column("SQL_ID", ScalarType.createStringType())
+                        .column("SQL_TEXT", ScalarType.createStringType())
+                        .column("OUTLINE_TARGET", ScalarType.createStringType())
+                        .column("OUTLINE_DATA", ScalarType.createStringType()).build()))
             .build();
 
     private boolean fetchAllFe = false;