diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 1f9da86abfab5c..80d583a4cfd8a8 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -26,6 +26,8 @@ file(GLOB_RECURSE EXEC_FILES CONFIGURE_DEPENDS *.cpp) if (WITH_LZO) set(EXEC_FILES ${EXEC_FILES} lzo_decompressor.cpp + schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.cpp + schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h ) endif() diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 0940eadb4951fd..b3b5d1fe8ad133 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -62,6 +62,7 @@ #include "runtime/define_primitive_type.h" #include "runtime/fragment_mgr.h" #include "runtime/types.h" +#include "schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h" #include "util/string_util.h" #include "util/types.h" #include "vec/columns/column.h" @@ -231,6 +232,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaBackendKerberosTicketCacheScanner::create_unique(); case TSchemaTableType::SCH_ROUTINE_LOAD_JOBS: return SchemaRoutineLoadJobScanner::create_unique(); + case TSchemaTableType::SCH_OPTIMIZER_SQL_PLAN_OUTLINE: + return SchemaOptimizerSqlPlanOutlineScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index 13670444e53c35..cd411a201734b6 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -151,4 +151,13 @@ Status SchemaHelper::fetch_routine_load_job(const std::string& ip, const int32_t }); } +Status SchemaHelper::fetch_outline_info(const std::string& ip, const int32_t port, + const TFetchOutlineInfoRequest& request, + TFetchOutlineInfoResult* result) { + return ThriftRpcHelper::rpc( + ip, port, [&request, &result](FrontendServiceConnection& client) { + client->fetchOutlineInfo(*result, request); + }); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 53d37389371a9f..7a9ea465901159 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -26,6 +26,8 @@ namespace doris { class TDescribeTablesParams; class TDescribeTablesResult; +class TFetchOutlineInfoRequest; +class TFetchOutlineInfoResult; class TFetchRoutineLoadJobRequest; class TFetchRoutineLoadJobResult; class TGetDbsParams; @@ -90,6 +92,10 @@ class SchemaHelper { static Status fetch_routine_load_job(const std::string& ip, const int32_t port, const TFetchRoutineLoadJobRequest& request, TFetchRoutineLoadJobResult* result); + + static Status fetch_outline_info(const std::string& ip, const int32_t port, + const TFetchOutlineInfoRequest& request, + TFetchOutlineInfoResult* result); }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.cpp b/be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.cpp new file mode 100644 index 00000000000000..3270984fcc5f5a --- /dev/null +++ b/be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.cpp @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h" + +#include +#include + +#include + +#include "exec/schema_scanner/schema_helper.h" +#include "runtime/runtime_state.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_factory.hpp" + +namespace doris { +class RuntimeState; +namespace vectorized { +class Block; +} // namespace vectorized + +std::vector SchemaOptimizerSqlPlanOutlineScanner::_s_tbls_columns = { + // name, type, size, is_null + {"OUTLINE_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"VISIBLE_SIGNATURE", TYPE_STRING, sizeof(StringRef), true}, + {"SQL_ID", TYPE_STRING, sizeof(StringRef), true}, + {"SQL_TEXT", TYPE_STRING, sizeof(StringRef), true}, + {"OUTLINE_TARGET", TYPE_STRING, sizeof(StringRef), true}, + {"OUTLINE_DATA", TYPE_STRING, sizeof(StringRef), true}, +}; + +SchemaOptimizerSqlPlanOutlineScanner::SchemaOptimizerSqlPlanOutlineScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_OPTIMIZER_SQL_PLAN_OUTLINE) {} + +SchemaOptimizerSqlPlanOutlineScanner::~SchemaOptimizerSqlPlanOutlineScanner() {} + +Status SchemaOptimizerSqlPlanOutlineScanner::start(RuntimeState* state) { + if (!_is_init) { + return Status::InternalError("used before initialized."); + } + TFetchOutlineInfoRequest request; + RETURN_IF_ERROR(SchemaHelper::fetch_outline_info( + *(_param->common_param->ip), _param->common_param->port, request, &_result)); + return Status::OK(); +} + +Status SchemaOptimizerSqlPlanOutlineScanner::get_next_block_internal(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (block == nullptr || eos == nullptr) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + if (_result.outlineInfos.empty()) { + return Status::OK(); + } + + return _fill_block_impl(block); +} + +Status SchemaOptimizerSqlPlanOutlineScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + + const auto& outline_infos = _result.outlineInfos; + size_t row_num = outline_infos.size(); + if (row_num == 0) { + return Status::OK(); + } + + for (size_t col_idx = 0; col_idx < _s_tbls_columns.size(); ++col_idx) { + const auto& col_desc = _s_tbls_columns[col_idx]; + + std::vector str_refs(row_num); + std::vector int_vals(row_num); + std::vector bool_vals(row_num); + std::vector datas(row_num); + std::vector column_values(row_num); + + for (size_t row_idx = 0; row_idx < row_num; ++row_idx) { + const auto& outline_info = outline_infos[row_idx]; + std::string& column_value = column_values[row_idx]; + + if (col_desc.type == TYPE_STRING) { + switch (col_idx) { + case 0: // OUTLINE_NAME + column_value = outline_info.__isset.outline_name ? outline_info.outline_name : ""; + break; + case 1: // VISIBLE_SIGNATURE + column_value = outline_info.__isset.visible_signature ? outline_info.visible_signature : ""; + break; + case 2: // SQL_ID + column_value = outline_info.__isset.sql_id ? outline_info.sql_id : ""; + break; + case 3: // SQL_TEXT + column_value = outline_info.__isset.sql_text ? outline_info.sql_text : ""; + break; + case 4: // OUTLINE_TARGET + column_value = outline_info.__isset.outline_target ? outline_info.outline_target : ""; + break; + case 5: // OUTLINE_DATA + column_value = outline_info.__isset.outline_data ? outline_info.outline_data : ""; + break; + } + + str_refs[row_idx] = + StringRef(column_values[row_idx].data(), column_values[row_idx].size()); + datas[row_idx] = &str_refs[row_idx]; + } + } + + RETURN_IF_ERROR(fill_dest_column_for_range(block, col_idx, datas)); + } + + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h b/be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h new file mode 100644 index 00000000000000..1ec2f7b8daf131 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_sch_optimizer_sql_plan_outline_scanner.h @@ -0,0 +1,52 @@ +//SCHEMA_SCH_OPTIMIZER_SQL_PLAN_OUTLINE_SCANNER_H + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "exec/schema_scanner.h" + +namespace doris { + class RuntimeState; + namespace vectorized { + class Block; + } // namespace vectorized + + class SchemaOptimizerSqlPlanOutlineScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaOptimizerSqlPlanOutlineScanner); + + public: + SchemaOptimizerSqlPlanOutlineScanner(); + ~SchemaOptimizerSqlPlanOutlineScanner() override; + + Status start(RuntimeState* state) override; + Status get_next_block_internal(vectorized::Block* block, bool* eos) override; + + private: + Status _fill_block_impl(vectorized::Block* block); + + TFetchOutlineInfoResult _result; + static std::vector _s_tbls_columns; + }; + +} // namespace doris diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 68e85a39a826d6..a6b2c5701aa192 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -380,6 +380,7 @@ OR: 'OR'; ORDER: 'ORDER'; OUTER: 'OUTER'; OUTFILE: 'OUTFILE'; +OUTLINE: 'OUTLINE'; OVER: 'OVER'; OVERWRITE: 'OVERWRITE'; PARAMETER: 'PARAMETER'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index f2eada27362450..0624176d0e52c4 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -68,6 +68,7 @@ statementBase | supportedKillStatement #supportedKillStatementAlias | supportedStatsStatement #supportedStatsStatementAlias | supportedTransactionStatement #supportedTransactionStatementAlias + | supportedOutlineStatement #supportedOutlineStatementAlias | unsupportedStatement #unsupported ; @@ -374,6 +375,12 @@ supportedLoadStatement | createRoutineLoad #createRoutineLoadAlias ; +supportedOutlineStatement + : CREATE (OR REPLACE)? OUTLINE outline_name=identifierOrText ON (query (TO query)? | + sql_id=STRING_LITERAL USING HINT_START identifier HINT_END ) #createOutline + | DROP OUTLINE (IF EXISTS)? outline_name=identifierOrText #dropOutline + ; + supportedOtherStatement : HELP mark=identifierOrText #help | UNLOCK TABLES #unlockTables @@ -1980,6 +1987,7 @@ nonReserved | ONLY | OPEN | OPTIMIZED + | OUTLINE | PARAMETER | PARSED | PASSWORD diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index 7bb9e44412a272..62229278ef6884 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -91,8 +91,10 @@ public enum SchemaTableType { TSchemaTableType.SCH_CATALOG_META_CACHE_STATISTICS), SCH_BACKEND_KERBEROS_TICKET_CACHE("BACKEND_KERBEROS_TICKET_CACHE", "BACKEND_KERBEROS_TICKET_CACHE", TSchemaTableType.SCH_BACKEND_KERBEROS_TICKET_CACHE), - SCH_ROUTINE_LOAD_JOBS("ROUTINE_LOAD_JOBS", "ROUTINE_LOAD_JOBS", - TSchemaTableType.SCH_ROUTINE_LOAD_JOBS); + SCH_ROUTINE_LOAD_JOB("ROUTINE_LOAD_JOB", "ROUTINE_LOAD_JOB", + TSchemaTableType.SCH_ROUTINE_LOAD_JOBS), + SCH_OPTIMIZER_SQL_PLAN_OUTLINE("OPTIMIZER_SQL_PLAN_OUTLINE", "OPTIMIZER_SQL_PLAN_OUTLINE", + TSchemaTableType.SCH_OPTIMIZER_SQL_PLAN_OUTLINE); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index a3b0c300951cf4..e3bbbbca75f8b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -200,6 +200,7 @@ import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.hint.OutlineMgr; import org.apache.doris.nereids.jobs.load.LabelProcessor; import org.apache.doris.nereids.stats.HboPlanStatisticsManager; import org.apache.doris.nereids.trees.plans.commands.AlterSystemCommand; @@ -391,6 +392,7 @@ public class Env { private StreamLoadRecordMgr streamLoadRecordMgr; private TabletLoadIndexRecorderMgr tabletLoadIndexRecorderMgr; private RoutineLoadManager routineLoadManager; + private OutlineMgr outlineMgr; private GroupCommitManager groupCommitManager; private SqlBlockRuleMgr sqlBlockRuleMgr; private ExportMgr exportMgr; @@ -713,6 +715,7 @@ public Env(boolean isCheckpointCatalog) { this.catalogMgr = new CatalogMgr(); this.load = new Load(); this.routineLoadManager = EnvFactory.getInstance().createRoutineLoadManager(); + this.outlineMgr = new OutlineMgr(); this.groupCommitManager = new GroupCommitManager(); this.sqlBlockRuleMgr = new SqlBlockRuleMgr(); this.exportMgr = new ExportMgr(); @@ -2403,6 +2406,13 @@ public long loadGlobalVariable(DataInputStream in, long checksum) throws IOExcep return checksum; } + // outline persistence + public long loadOutline(DataInputStream in, long checksum) throws IOException, DdlException { + VariableMgr.read(in); + LOG.info("finished replay globalVariable from image"); + return checksum; + } + // load binlogs public long loadBinlogs(DataInputStream dis, long checksum) throws IOException { binlogManager.read(dis, checksum); @@ -2422,6 +2432,12 @@ public long loadRoutineLoadJobs(DataInputStream dis, long checksum) throws IOExc return checksum; } + public long loadOutlines(DataInputStream dis, long checksum) throws IOException { + Env.getCurrentEnv().getOutlineMgr().readFields(dis); + LOG.info("finished replay outlines from image"); + return checksum; + } + public long loadLoadJobsV2(DataInputStream in, long checksum) throws IOException { loadManager.readFields(in); LOG.info("finished replay loadJobsV2 from image"); @@ -2730,6 +2746,11 @@ public long saveRoutineLoadJobs(CountingDataOutputStream dos, long checksum) thr return checksum; } + public long saveOutlines(CountingDataOutputStream dos, long checksum) throws IOException { + Env.getCurrentEnv().getOutlineMgr().write(dos); + return checksum; + } + public long saveGlobalVariable(CountingDataOutputStream dos, long checksum) throws IOException { VariableMgr.write(dos); return checksum; @@ -4658,6 +4679,10 @@ public RoutineLoadManager getRoutineLoadManager() { return routineLoadManager; } + public OutlineMgr getOutlineMgr() { + return outlineMgr; + } + public GroupCommitManager getGroupCommitManager() { return groupCommitManager; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 13c44f2fed81b0..174c8cf17110a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -620,7 +620,14 @@ public class SchemaTable extends Table { .column("CURRENT_ABORT_TASK_NUM", ScalarType.createType(PrimitiveType.INT)) .column("IS_ABNORMAL_PAUSE", ScalarType.createType(PrimitiveType.BOOLEAN)) .build()) - ) + ).put("optimizer_sql_plan_outline", + new SchemaTable(SystemIdGenerator.getNextId(), "optimizer_sql_plan_outline", TableType.SCHEMA, + builder().column("OUTLINE_NAME", ScalarType.createStringType()) + .column("VISIBLE_SIGNATURE", ScalarType.createStringType()) + .column("SQL_ID", ScalarType.createStringType()) + .column("SQL_TEXT", ScalarType.createStringType()) + .column("OUTLINE_TARGET", ScalarType.createStringType()) + .column("OUTLINE_DATA", ScalarType.createStringType()).build())) .build(); private boolean fetchAllFe = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index d0416ba5f6d26b..52293784a1fad8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -73,6 +73,7 @@ public class SummaryProfile { public static final String DISTRIBUTED_PLAN = "Distributed Plan"; public static final String SYSTEM_MESSAGE = "System Message"; public static final String EXECUTED_BY_FRONTEND = "Executed By Frontend"; + public static final String OUTLINE_NAME = "Outline Name"; // Execution Summary public static final String EXECUTION_SUMMARY_PROFILE_NAME = "Execution Summary"; public static final String ANALYSIS_TIME = "Analysis Time"; @@ -133,7 +134,8 @@ public class SummaryProfile { // a column, so that should not // add many columns here. Add to ExecutionSummary list. public static final ImmutableList SUMMARY_CAPTIONS = ImmutableList.of(PROFILE_ID, TASK_TYPE, - START_TIME, END_TIME, TOTAL_TIME, TASK_STATE, USER, DEFAULT_CATALOG, DEFAULT_DB, SQL_STATEMENT); + START_TIME, END_TIME, TOTAL_TIME, TASK_STATE, USER, DEFAULT_CATALOG, + DEFAULT_DB, SQL_STATEMENT, OUTLINE_NAME); public static final ImmutableList SUMMARY_KEYS = new ImmutableList.Builder() .addAll(SUMMARY_CAPTIONS) .add(DISTRIBUTED_PLAN) @@ -766,6 +768,11 @@ public SummaryBuilder sqlStatement(String val) { return this; } + public SummaryBuilder outlineName(String val) { + map.put(OUTLINE_NAME, val); + return this; + } + public SummaryBuilder isCached(String val) { map.put(IS_CACHED, val); return this; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 29096b68ce00bb..0f5e116a8a41b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -60,6 +60,7 @@ import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.sync.SyncJob; import org.apache.doris.mysql.privilege.UserPropertyInfo; +import org.apache.doris.nereids.hint.OutlineInfo; import org.apache.doris.persist.AlterConstraintLog; import org.apache.doris.persist.AlterDatabasePropertyInfo; import org.apache.doris.persist.AlterLightSchemaChangeInfo; @@ -955,6 +956,16 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_CREATE_OUTLINE: { + data = OutlineInfo.read(in); + isRead = true; + break; + } + case OperationType.OP_DROP_OUTLINE: { + data = OutlineInfo.read(in); + isRead = true; + break; + } // FIXME: support cloud related operation types. case OperationType.OP_UPDATE_CLOUD_REPLICA: { data = UpdateCloudReplicaInfo.read(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index ad5f4f3d0e3697..96f77d86018510 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -39,6 +39,8 @@ import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.hint.DistributeHint; import org.apache.doris.nereids.hint.Hint; +import org.apache.doris.nereids.hint.OutlineInfo; +import org.apache.doris.nereids.hint.OutlineMgr; import org.apache.doris.nereids.jobs.executor.Optimizer; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.memo.Group; @@ -47,6 +49,7 @@ import org.apache.doris.nereids.metrics.event.CounterEvent; import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.minidump.NereidsTracer; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.pre.PlanPreprocessors; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -144,6 +147,17 @@ public void plan(StatementBase queryStmt, org.apache.doris.thrift.TQueryOptions ExplainLevel explainLevel = getExplainLevel(queryStmt.getExplainOptions()); LogicalPlan parsedPlan = logicalPlanAdapter.getLogicalPlan(); + if (statementContext.getConnectContext().getSessionVariable().isEnableSqlPlanOutlines() + && queryStmt.getOrigStmt() != null) { + String visibleSignature = OutlineMgr.replaceConstant(queryStmt.getOrigStmt().originStmt, + statementContext.getConstantExpressionMap(), 0); + Optional outlineInfo = OutlineMgr.getOutlineByVisibleSignature(visibleSignature); + if (outlineInfo.isPresent()) { + NereidsParser parser = new NereidsParser(); + parsedPlan = parser.parseForOutline(outlineInfo.get().getOutlineData(), parsedPlan); + queryStmt.getOrigStmt().setOutlineName(outlineInfo.get().getOutlineName()); + } + } NereidsTracer.logImportantTime("EndParsePlan"); setParsedPlan(parsedPlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index cc09e260a8857c..35c038d7f56225 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -218,6 +218,8 @@ public enum TableFrom { private boolean prepareStage = false; + private Map> constantExpressionMap = new HashMap<>(); + public StatementContext() { this(ConnectContext.get(), null, 0); } @@ -250,6 +252,10 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta } } + public Map> getConstantExpressionMap() { + return constantExpressionMap; + } + public void setNeedLockTables(boolean needLockTables) { this.needLockTables = needLockTables; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OutlineInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OutlineInfo.java new file mode 100644 index 00000000000000..eb8f3b7f2084c2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OutlineInfo.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.hint; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * outline information which outline should have + */ +public class OutlineInfo implements Writable { + @SerializedName("outlineName") + private final String outlineName; + @SerializedName("visibleSignature") + private final String visibleSignature; + @SerializedName("sqlId") + private final String sqlId; + @SerializedName("sqlText") + private final String sqlText; + @SerializedName("outlineTarget") + private final String outlineTarget; + @SerializedName("outlineData") + private final String outlineData; + + /** + * outline information which outline should have + * @param outlineName name of outline, also as a key in outline map + * @param visibleSignature sql we want to add outline after change constant to param + * @param sqlId sql id use visible signature as hash key + * @param sqlText sql we want to add outline before change constant to param + * @param outlineTarget sql we want to add outline with syntax on + * @param outlineData outline data which we need to add to matched sqls + */ + public OutlineInfo(String outlineName, String visibleSignature, String sqlId, String sqlText, String outlineTarget, + String outlineData) { + this.outlineName = outlineName; + this.visibleSignature = visibleSignature; + this.sqlId = sqlId; + this.sqlText = sqlText; + this.outlineTarget = outlineTarget; + this.outlineData = outlineData; + } + + public String getOutlineName() { + return outlineName; + } + + public String getVisibleSignature() { + return visibleSignature; + } + + public String getOutlineData() { + return outlineData; + } + + public String getOutlineTarget() { + return outlineTarget; + } + + public String getSqlId() { + return sqlId; + } + + public String getSqlText() { + return sqlText; + } + + public static OutlineInfo read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), OutlineInfo.class); + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OutlineMgr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OutlineMgr.java new file mode 100644 index 00000000000000..35f03a189c9227 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OutlineMgr.java @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.hint; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; +import org.apache.doris.common.io.Writable; +import org.apache.doris.nereids.trees.plans.PlaceholderId; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.VariableMgr; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Outline manager used to manage read write and cached of outline + */ +public class OutlineMgr implements Writable { + public static final OutlineMgr INSTANCE = new OutlineMgr(); + private static final Logger LOG = LogManager.getLogger(OutlineMgr.class); + private static final Map outlineMap = new HashMap<>(); + private static final Map visibleSignatureMap = new HashMap<>(); + private static final HashSet nereidsVars = new HashSet<>(Arrays.asList( + "DISABLE_NEREIDS_RULES", "ALLOW_MODIFY_MATERIALIZED_VIEW_DATA", "ALLOW_PARTITION_COLUMN_NULLABLE", + "AUTO_BROADCAST_JOIN_THRESHOLD", "AUTO_INCREMENT_INCREMENT", "AUTO_PROFILE_THRESHOLD_MS", + "BROADCAST_HASHTABLE_MEM_LIMIT_PERCENTAGE", "BROADCAST_RIGHT_TABLE_SCALE_FACTOR", + "CBO_CPU_WEIGHT", "CBO_MEM_WEIGHT", "CBO_NET_WEIGHT", "CHECK_OVERFLOW_FOR_DECIMAL", + "DECIMAL_OVERFLOW_SCALE", "DISABLE_COLOCATE_PLAN", "DISABLE_EMPTY_PARTITION_PRUNE", + "DISABLE_JOIN_REORDER", "DUMP_NEREIDS_MEMO", "ENABLE_BUCKET_SHUFFLE_JOIN", "ENABLE_BUSHY_TREE", + "ENABLE_CBO_STATISTICS", "ENABLE_COMMON_EXPR_PUSHDOWN", "ENABLE_COMMON_EXPR_PUSHDOWN_FOR_INVERTED_INDEX", + "ENABLE_COOLDOWN_REPLICA_AFFINITY", "ENABLE_COST_BASED_JOIN_REORDER", "ENABLE_CTE_MATERIALIZE", + "ENABLE_DECIMAL256", "ENABLE_DML_MATERIALIZED_VIEW_REWRITE", "BROADCAST_ROW_COUNT_LIMIT", + "ENABLE_DML_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS", + "ENABLE_DPHYP_OPTIMIZER", "ENABLE_DPHYP_TRACE", "ENABLE_ELIMINATE_SORT_NODE", "ENABLE_EXPR_TRACE", + "ENABLE_EXT_FUNC_PRED_PUSHDOWN", "ENABLE_FAST_ANALYZE_INTO_VALUES", "ENABLE_FOLD_CONSTANT_BY_BE", + "ENABLE_FOLD_NONDETERMINISTIC_FN", "ENABLE_FUNCTION_PUSHDOWN", "ENABLE_HASH_JOIN_EARLY_START_PROBE", + "ENABLE_INFER_PREDICATE", "ENABLE_INSERT_STRICT", "ENABLE_INVERTED_INDEX_QUERY", + "ENABLE_INVERTED_INDEX_QUERY_CACHE", "ENABLE_INVERTED_INDEX_SEARCHER_CACHE", "ENABLE_JOIN_SPILL", + "ENABLE_LEFT_ZIG_ZAG", "ENABLE_LOCAL_MERGE_SORT", "ENABLE_MATERIALIZED_VIEW_NEST_REWRITE", + "ENABLE_MATERIALIZED_VIEW_REWRITE", "ENABLE_NEREIDS_RULES", "ENABLE_NEREIDS_TIMEOUT", + "ENABLE_NEREIDS_TRACE", "ENABLE_NEW_COST_MODEL", "ENABLE_ORDERED_SCAN_RANGE_LOCATIONS", + "ENABLE_PARTITION_TOPN", "ENABLE_PROFILE", "ENABLE_PUSH_DOWN_NO_GROUP_AGG", + "ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE", "DEBUG_SKIP_FOLD_CONSTANT", + "ENABLE_PUSHDOWN_STRING_MINMAX", "ENABLE_QUERY_CACHE", "ENABLE_REWRITE_ELEMENT_AT_TO_SLOT", + "ENABLE_RUNTIME_FILTER_PARTITION_PRUNE", "ENABLE_RUNTIME_FILTER_PRUNE", + "ENABLE_SHARED_EXCHANGE_SINK_BUFFER", "ENABLE_SQL_CACHE", "ENABLE_STATS", + "ENABLE_SYNC_MV_COST_BASED_REWRITE", "ENABLE_VERBOSE_PROFILE", "EXPAND_RUNTIME_FILTER_BY_INNER_JOIN", + "EXPERIMENTAL_ENABLE_AGG_STATE", "EXPERIMENTAL_ENABLE_COMPRESS_MATERIALIZE", + "EXPERIMENTAL_ENABLE_LOCAL_SHUFFLE", "ENABLE_SHARE_HASH_TABLE_FOR_BROADCAST_JOIN", + "EXPERIMENTAL_ENABLE_NEREIDS_DISTRIBUTE_PLANNER", "EXPERIMENTAL_ENABLE_PARALLEL_SCAN", + "EXPERIMENTAL_ENABLE_SHARED_SCAN", "EXPERIMENTAL_FORCE_TO_LOCAL_SHUFFLE", "FILTER_COST_FACTOR", + "FORBID_UNKNOWN_COL_STATS", "FORWARD_TO_MASTER", "GLOBAL_PARTITION_TOPN_THRESHOLD", + "IGNORE_RUNTIME_FILTER_ERROR", "IGNORE_RUNTIME_FILTER_IDS", "INLINE_CTE_REFERENCED_THRESHOLD", + "INSERT_MAX_FILTER_RATIO", "INSERT_TIMEOUT", "INSERT_VISIBLE_TIMEOUT_MS", + "INTERACTIVE_TIMEOUT", "JOIN_ORDER_TIME_LIMIT", "LEFT_SEMI_OR_ANTI_PROBE_FACTOR", + "MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT", "MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE", + "MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM", "MAX_EXECUTION_TIME", + "MAX_JOIN_NUMBER_BUSHY_TREE", "MAX_JOIN_NUMBER_OF_REORDER", "MAX_PUSHDOWN_CONDITIONS_PER_COLUMN", + "MAX_TABLE_COUNT_USE_CASCADES_JOIN_REORDER", "MEMO_MAX_GROUP_EXPRESSION_SIZE", + "NEREIDS_CBO_PENALTY_FACTOR", "NEREIDS_STAR_SCHEMA_SUPPORT", "NEREIDS_TIMEOUT_SECOND", + "NEREIDS_TRACE_EVENT_MODE", "NTH_OPTIMIZED_PLAN", "PARTITION_PRUNING_EXPAND_THRESHOLD", + "PLAN_NEREIDS_DUMP", "PREFER_JOIN_METHOD", "PROFILE_LEVEL", "PROFILING", "PUSH_TOPN_TO_AGG", + "QUERY_CACHE_ENTRY_MAX_BYTES", "QUERY_CACHE_ENTRY_MAX_ROWS", "QUERY_CACHE_FORCE_REFRESH", + "QUERY_CACHE_TYPE", "QUERY_TIMEOUT", "REQUIRE_SEQUENCE_IN_INSERT", "REWRITE_COUNT_DISTINCT_TO_BITMAP_HLL", + "REWRITE_OR_TO_IN_PREDICATE_THRESHOLD", "ROUND_PRECISE_DECIMALV2_VALUE", "RUNTIME_BLOOM_FILTER_MAX_SIZE", + "RUNTIME_BLOOM_FILTER_MIN_SIZE", "RUNTIME_BLOOM_FILTER_SIZE", "RUNTIME_FILTER_JUMP_THRESHOLD", + "RUNTIME_FILTER_MAX_IN_NUM", "RUNTIME_FILTER_MODE", "RUNTIME_FILTER_PRUNE_FOR_EXTERNAL", + "RUNTIME_FILTER_TYPE", "RUNTIME_FILTER_WAIT_INFINITELY", "RUNTIME_FILTER_WAIT_TIME_MS", + "RUNTIME_FILTERS_MAX_NUM", "SHOW_ALL_FE_CONNECTION", "SQL_DIALECT", "SQL_SELECT_LIMIT", + "STORAGE_ENGINE", "TIME_ZONE", "TOPN_FILTER_RATIO", "TOPN_OPT_LIMIT_THRESHOLD", + "TRACE_NEREIDS", "USE_MAX_LENGTH_OF_VARCHAR_IN_CTAS", "USE_RF_DEFAULT", "WAIT_TIMEOUT")); + + private static ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + private void readLock() { + lock.readLock().lock(); + } + + private void readUnlock() { + lock.readLock().unlock(); + } + + private static void writeLock() { + lock.writeLock().lock(); + } + + private static void writeUnlock() { + lock.writeLock().unlock(); + } + + public static Optional getOutline(String outlineName) { + if (outlineMap.containsKey(outlineName)) { + return Optional.of(outlineMap.get(outlineName)); + } + return Optional.empty(); + } + + public static Map getOutlineMap() { + return outlineMap; + } + + public static Optional getOutlineByVisibleSignature(String visibleSignature) { + if (visibleSignatureMap.containsKey(visibleSignature)) { + return Optional.of(visibleSignatureMap.get(visibleSignature)); + } + return Optional.empty(); + } + + public static Map getVisibleSignatureMap() { + return visibleSignatureMap; + } + + /** + * replace constant by place holder + * @param originalQuery original query input by create outline command + * @param constantMap constant map collected by logicalPlanBuilder + * @param startIndex a shift of create outline command + * @return query replace constant by place holder + */ + public static String replaceConstant(String originalQuery, Map> constantMap, int startIndex) { + List> sortedKeys = new ArrayList<>(constantMap.values()); + + // Sort by start index in descending order to avoid shifting problems + sortedKeys.sort((a, b) -> b.first.compareTo(a.first)); + + StringBuilder sb = new StringBuilder(originalQuery); + + for (Pair range : sortedKeys) { + if (range.first == 0 && range.second == 0) { + continue; + } + int start = range.first - startIndex; + int end = range.second - startIndex + 1; + sb.replace(start, end, "?"); + } + + return sb.toString(); + } + + /** + * create outline data which include some hints + * @param sessionVariable sessionVariables used to generate corresponding plan + * @return string include many hints + */ + public static String createOutlineData(SessionVariable sessionVariable) { + StringBuilder sb = new StringBuilder(); + sb.append("/*+ "); + // add set_var hint + List> changedVars = VariableMgr.dumpNereidsVars(sessionVariable, nereidsVars); + if (!changedVars.isEmpty()) { + sb.append("set_var("); + for (List changedVar : changedVars) { + sb.append(changedVar.get(0)); + sb.append("="); + sb.append("\""); + sb.append(changedVar.get(1)); + sb.append("\""); + sb.append("\t"); + } + sb.append(") "); + } + sb.append("*/ "); + return sb.toString(); + } + + /** + * createOutlineInternal + * @param outlineInfo outline info used to create outline + * @param ignoreIfExists if we add or replace to create outline statement, it would be true + * @param isReplay when it is replay mode, editlog would not be written + * @throws DdlException should throw exception when meeting problem + */ + public static void createOutlineInternal(OutlineInfo outlineInfo, boolean ignoreIfExists, boolean isReplay) + throws DdlException { + writeLock(); + try { + if (!ignoreIfExists && OutlineMgr.getOutline(outlineInfo.getOutlineName()).isPresent()) { + LOG.info("outline already exists, ignored to create outline: {}, is replay: {}", + outlineInfo.getOutlineName(), isReplay); + throw new DdlException(outlineInfo.getOutlineName() + " already exists"); + } + + if (OutlineMgr.getOutlineByVisibleSignature(outlineInfo.getVisibleSignature()).isPresent()) { + if (!ignoreIfExists) { + LOG.info("outline already exists, ignored to create outline with signature: {}, is replay: {}", + outlineInfo.getVisibleSignature(), isReplay); + throw new DdlException(outlineInfo.getVisibleSignature() + " already exists"); + } else { + dropOutline(outlineInfo); + } + } + + createOutline(outlineInfo); + if (!isReplay) { + Env.getCurrentEnv().getEditLog().logCreateOutline(outlineInfo); + } + } finally { + writeUnlock(); + } + LOG.info("finished to create outline: {}, is replay: {}", outlineInfo.getOutlineName(), isReplay); + } + + private static void createOutline(OutlineInfo outlineInfo) { + outlineMap.put(outlineInfo.getOutlineName(), outlineInfo); + visibleSignatureMap.put(outlineInfo.getVisibleSignature(), outlineInfo); + } + + /** + * createOutlineInternal + * @param outlineName outline info used to create outline + * @param ifExists if we add if exists to create outline statement, it would be true + * @param isReplay when it is replay mode, editlog would not be written + * @throws DdlException should throw exception when meeting problem + */ + public static void dropOutlineInternal(String outlineName, boolean ifExists, boolean isReplay) + throws DdlException { + writeLock(); + try { + boolean isPresent = outlineMap.containsKey(outlineName); + if (!isPresent) { + LOG.info("outline not exists, ignored to drop outline: {}, is replay: {}", + outlineName, isReplay); + if (!ifExists) { + throw new DdlException(outlineName + " not exists"); + } else { + return; + } + } + + OutlineInfo outlineInfo = outlineMap.get(outlineName); + dropOutline(outlineInfo); + if (!isReplay && isPresent) { + Env.getCurrentEnv().getEditLog().logDropOutline(outlineInfo); + } + } finally { + writeUnlock(); + } + LOG.info("finished to create outline: {}, is replay: {}", outlineName, isReplay); + } + + private static void dropOutline(OutlineInfo outlineInfo) { + outlineMap.remove(outlineInfo.getOutlineName()); + visibleSignatureMap.remove(outlineInfo.getVisibleSignature()); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(outlineMap.size()); + for (OutlineInfo outlineInfo : outlineMap.values()) { + outlineInfo.write(out); + } + } + + /** + * read fields from disk + * @param in data source of disk + * @throws IOException maybe throw ioexception + */ + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + OutlineInfo outlineInfo = OutlineInfo.read(in); + outlineMap.put(outlineInfo.getOutlineName(), outlineInfo); + visibleSignatureMap.put(outlineInfo.getVisibleSignature(), outlineInfo); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 85a443ae3441f1..5c3ea973fc34e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -570,6 +570,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateOutlineCommand; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; @@ -596,6 +597,7 @@ import org.apache.doris.nereids.trees.plans.commands.DropFunctionCommand; import org.apache.doris.nereids.trees.plans.commands.DropJobCommand; import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand; +import org.apache.doris.nereids.trees.plans.commands.DropOutlineCommand; import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.DropRepositoryCommand; import org.apache.doris.nereids.trees.plans.commands.DropResourceCommand; @@ -1962,7 +1964,44 @@ public LogicalPlan visitCreateRoutineLoad(CreateRoutineLoadContext ctx) { CreateRoutineLoadInfo createRoutineLoadInfo = new CreateRoutineLoadInfo(jobLabelInfo, tableName, loadPropertyMap, properties, type, customProperties, mergeType, comment); return new CreateRoutineLoadCommand(createRoutineLoadInfo); + } + + @Override + public Command visitCreateOutline(DorisParser.CreateOutlineContext ctx) { + String outlineName = stripQuotes(ctx.outline_name.getText()); + boolean isReplace = ctx.REPLACE() != null; + if (ctx.query().size() == 0) { + // with sql_id + } else if (ctx.query().size() == 1) { + // only on query is declared + LogicalPlan logicalPlan = visitQuery(ctx.query().get(0)); + String originalQuery = ctx.query().get(0).start.getInputStream() + .getText(new org.antlr.v4.runtime.misc.Interval(ctx.query().get(0).start.getStartIndex(), + ctx.query().get(0).stop.getStopIndex())); + int startIndex = ctx.query().get(0).start.getStartIndex(); + return new CreateOutlineCommand(outlineName, isReplace, logicalPlan, originalQuery, startIndex, ""); + } else if (ctx.query().size() == 2) { + // on query to query + LogicalPlan logicalPlan = visitQuery(ctx.query().get(0)); + String originalQuery = ctx.query().get(0).start.getInputStream() + .getText(new org.antlr.v4.runtime.misc.Interval(ctx.query().get(0).start.getStartIndex(), + ctx.query().get(0).stop.getStopIndex())); + int startIndex = ctx.query().get(0).start.getStartIndex(); + String targetQuery = ctx.query().get(1).start.getInputStream() + .getText(new org.antlr.v4.runtime.misc.Interval(ctx.query().get(1).start.getStartIndex(), + ctx.query().get(1).stop.getStopIndex())); + return new CreateOutlineCommand(outlineName, isReplace, logicalPlan, + originalQuery, startIndex, targetQuery); + } + + return new CreateOutlineCommand(outlineName, isReplace, null, "originalQuery", 0, ""); + } + @Override + public Command visitDropOutline(DorisParser.DropOutlineContext ctx) { + String outlineName = stripQuotes(ctx.outline_name.getText()); + boolean isExists = ctx.EXISTS() != null; + return new DropOutlineCommand(isExists, outlineName); } @Override @@ -3572,7 +3611,17 @@ private String toStringValue(String literal) { * visitor and only takes care of typing (We assume that the visitor returns an Expression here). */ private Expression getExpression(ParserRuleContext ctx) { - return typedVisit(ctx); + Expression result = typedVisit(ctx); + if (result instanceof Literal) { + Placeholder parameter = new Placeholder(ConnectContext.get().getStatementContext().getNextPlaceholderId()); + tokenPosToParameters.put(ctx.start, parameter); + ConnectContext.get().getStatementContext().getIdToPlaceholderRealExpr().put(parameter.getPlaceholderId(), + result); + ConnectContext.get().getStatementContext().getConstantExpressionMap() + .put(parameter.getPlaceholderId(), Pair.of(ctx.start.getStartIndex(), ctx.start.getStopIndex())); + return parameter; + } + return result; } private LogicalPlan withExplain(LogicalPlan inputPlan, ExplainContext ctx) { @@ -3790,6 +3839,12 @@ private List> getTableList(List ctx) { return tableList; } + /** + * with select hint need to be used to create logicalSelectHint logical plan + * @param logicalPlan child of select hint plan + * @param hintContexts hints used to be contexts of logicalSelectHint + * @return logicalSelectHint with child plan + */ private LogicalPlan withHints(LogicalPlan logicalPlan, List selectHintContexts, List preAggOnHintContexts) { if (selectHintContexts.isEmpty() && preAggOnHintContexts.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index df7b0b012af614..8fcc45fee40222 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -55,6 +55,7 @@ import org.apache.logging.log4j.Logger; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.BitSet; import java.util.Iterator; import java.util.List; @@ -343,6 +344,21 @@ public LogicalPlan parseForCreateView(String sql) { return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } + /** + * parser used for outline + * @param hintSql hint sql + * @param logicalPlan child plan + * @return logicalSelectHint plan with child plan + */ + public LogicalPlan parseForOutline(String hintSql, LogicalPlan logicalPlan) { + CommonTokenStream tokenStream = parseAllTokens(hintSql); + LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilder( + getHintMap(hintSql, tokenStream, DorisParser::selectHint)); + List selectHintContexts = new ArrayList<>(getHintMap(hintSql, + tokenStream, DorisParser::selectHint).values()); + return realLogicalPlanBuilder.withSelectHint(logicalPlan, selectHintContexts); + } + public LogicalPlan parseForEncryption(String sql, Map, String> indexInSqlToString) { CommonTokenStream tokenStream = parseAllTokens(sql); ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 029d44b8ed73d1..2f98867ef45d3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -513,7 +514,7 @@ private LogicalSort bindSortWithSetOperation( sort, cascadesContext, sort.children(), true, true); Builder boundKeys = ImmutableList.builderWithExpectedSize(sort.getOrderKeys().size()); for (OrderKey orderKey : sort.getOrderKeys()) { - Expression boundKey = bindWithOrdinal(orderKey.getExpr(), analyzer, childOutput); + Expression boundKey = bindWithOrdinal(orderKey.getExpr(), cascadesContext, analyzer, childOutput); boundKeys.add(orderKey.withExpression(boundKey)); } return new LogicalSort<>(boundKeys.build(), sort.child()); @@ -1026,7 +1027,7 @@ private List bindGroupBy( ImmutableList.Builder boundGroupByBuilder = ImmutableList.builderWithExpectedSize(groupBy.size()); for (Expression key : groupBy) { - boundGroupByBuilder.add(bindWithOrdinal(key, analyzer, boundAggOutput)); + boundGroupByBuilder.add(bindWithOrdinal(key, cascadesContext, analyzer, boundAggOutput)); } List boundGroupBy = boundGroupByBuilder.build(); checkIfOutputAliasNameDuplicatedForGroupBy(boundGroupBy, boundAggOutput); @@ -1115,7 +1116,8 @@ private Plan bindSortWithoutSetOperation(MatchingContext> ctx) if (hasAggregateFunction(orderKey.getExpr(), functionRegistry)) { boundKey = bindInInputChildScope.analyze(orderKey.getExpr()); } else { - boundKey = bindWithOrdinal(orderKey.getExpr(), bindInInputScopeThenInputChildScope, childOutput); + boundKey = bindWithOrdinal(orderKey.getExpr(), cascadesContext, + bindInInputScopeThenInputChildScope, childOutput); } boundOrderKeys.add(orderKey.withExpression(boundKey)); } @@ -1194,7 +1196,24 @@ private boolean isAggregateFunction(UnboundFunction unboundFunction, FunctionReg } private Expression bindWithOrdinal( - Expression unbound, SimpleExprAnalyzer analyzer, List boundSelectOutput) { + Expression unbound, CascadesContext cascadesContext, SimpleExprAnalyzer analyzer, + List boundSelectOutput) { + if (unbound instanceof Placeholder) { + Expression realExpr = cascadesContext.getStatementContext() + .getIdToPlaceholderRealExpr().get(((Placeholder) unbound).getPlaceholderId()); + if (realExpr instanceof IntegerLikeLiteral) { + cascadesContext.getStatementContext().getConstantExpressionMap() + .put(((Placeholder) unbound).getPlaceholderId(), Pair.of(0, 0)); + int ordinal = ((IntegerLikeLiteral) realExpr).getIntValue(); + if (ordinal >= 1 && ordinal <= boundSelectOutput.size()) { + Expression boundSelectItem = boundSelectOutput.get(ordinal - 1); + return boundSelectItem instanceof Alias ? boundSelectItem.child(0) : boundSelectItem; + } else { + return realExpr; // bound literal + } + } + } + if (unbound instanceof IntegerLikeLiteral) { int ordinal = ((IntegerLikeLiteral) unbound).getIntValue(); if (ordinal >= 1 && ordinal <= boundSelectOutput.size()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 0d60298421a3cd..a36f56ef2e61e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -306,6 +306,8 @@ public enum PlanType { CREATE_CATALOG_COMMAND, CREATE_FILE_COMMAND, CREATE_ROUTINE_LOAD_COMMAND, + CREATE_OUTLINE_COMMAND, + DROP_OUTLINE_COMMAND, SHOW_TABLE_CREATION_COMMAND, SHOW_QUERY_PROFILE_COMMAND, SWITCH_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateOutlineCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateOutlineCommand.java new file mode 100644 index 00000000000000..e35a9e9e96d595 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateOutlineCommand.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.StmtType; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.hint.OutlineInfo; +import org.apache.doris.nereids.hint.OutlineMgr; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.base.Strings; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * create outline command + */ +public class CreateOutlineCommand extends Command implements ForwardWithSync { + public static final Logger LOG = LogManager.getLogger(CreateOutlineCommand.class); + + private final String outlineName; + + private final Boolean ignoreExist; + + private final LogicalPlan query; + + private final String originalQuery; + + private final int startIndex; + + private final String targetQuery; + + /** + * create outline command + * @param outlineName outline name which is unique + * @param ignoreExist if true create anyway + * @param query query behind on expression parsed into logical plan + * @param originalQuery original query string + * @param startIndex start index of original query + */ + public CreateOutlineCommand(String outlineName, boolean ignoreExist, + LogicalPlan query, String originalQuery, int startIndex, String targetQuery) { + super(PlanType.CREATE_OUTLINE_COMMAND); + this.outlineName = outlineName; + this.ignoreExist = ignoreExist; + this.query = query; + this.originalQuery = originalQuery; + this.startIndex = startIndex; + this.targetQuery = targetQuery; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!ctx.getSessionVariable().isEnableSqlPlanOutlines()) { + LOG.info("outline is not enabled, please enable it by: set enable_sql_plan_outlines = true"); + throw new AnalysisException( + "outline is not enabled, please enable it by: set enable_sql_plan_outlines = true"); + } + NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(query, ctx.getStatementContext()); + executor.setParsedStmt(logicalPlanAdapter); + if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) { + ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE); + } + logicalPlanAdapter.setOrigStmt(new OriginStatement(originalQuery, 0)); + ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_SQL_PLAN_OUTLINES, "false"); + planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); + executor.setPlanner(planner); + executor.checkBlockRules(); + String visibleSignature; + if (Strings.isNullOrEmpty(targetQuery)) { + visibleSignature = OutlineMgr.replaceConstant(originalQuery, + executor.getContext().getStatementContext().getConstantExpressionMap(), startIndex); + } else { + if (!targetQuery.equals(originalQuery)) { + LOG.info("target query should be the same as the original query"); + throw new AnalysisException("target query should be the same as the original query"); + } + visibleSignature = targetQuery; + } + String sqlId = DigestUtils.md5Hex(visibleSignature); + String outlineData = OutlineMgr.createOutlineData(ctx.getSessionVariable()); + OutlineInfo outlineInfo = new OutlineInfo(outlineName, visibleSignature, sqlId, + originalQuery, targetQuery, outlineData); + OutlineMgr.createOutlineInternal(outlineInfo, ignoreExist, false); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateOutlineCommand(this, context); + } + + @Override + public StmtType stmtType() { + return StmtType.CREATE; + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropOutlineCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropOutlineCommand.java new file mode 100644 index 00000000000000..7e8c826575481c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropOutlineCommand.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.StmtType; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +/** + * base class for all drop commands + */ +public class DropOutlineCommand extends Command implements ForwardWithSync { + private final boolean ifExists; + private final String outlineName; + + public DropOutlineCommand(boolean ifExists, String outlineName) { + super(PlanType.DROP_OUTLINE_COMMAND); + this.ifExists = ifExists; + this.outlineName = outlineName; + } + + @Override + public StmtType stmtType() { + return StmtType.DROP; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitDropOutlineCommand(this, context); + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + ctx.getEnv().getOutlineMgr().dropOutlineInternal(outlineName, ifExists, false); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index a0d55d59777c2f..96f19f4dab998b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -56,6 +56,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateOutlineCommand; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; @@ -81,6 +82,7 @@ import org.apache.doris.nereids.trees.plans.commands.DropFunctionCommand; import org.apache.doris.nereids.trees.plans.commands.DropJobCommand; import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand; +import org.apache.doris.nereids.trees.plans.commands.DropOutlineCommand; import org.apache.doris.nereids.trees.plans.commands.DropProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.DropRepositoryCommand; import org.apache.doris.nereids.trees.plans.commands.DropResourceCommand; @@ -284,6 +286,14 @@ default R visitCreateTableCommand(CreateTableCommand createTableCommand, C conte return visitCommand(createTableCommand, context); } + default R visitCreateOutlineCommand(CreateOutlineCommand createOutlineCommand, C context) { + return visitCommand(createOutlineCommand, context); + } + + default R visitDropOutlineCommand(DropOutlineCommand dropOutlineCommand, C context) { + return visitCommand(dropOutlineCommand, context); + } + default R visitCreateMTMVCommand(CreateMTMVCommand createMTMVCommand, C context) { return visitCommand(createMTMVCommand, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 824caf78bd1edb..386f68d81ab850 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -81,6 +81,7 @@ import org.apache.doris.meta.MetaContext; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.privilege.UserPropertyInfo; +import org.apache.doris.nereids.hint.OutlineInfo; import org.apache.doris.plsql.metastore.PlsqlPackage; import org.apache.doris.plsql.metastore.PlsqlProcedureKey; import org.apache.doris.plsql.metastore.PlsqlStoredProcedure; @@ -535,6 +536,16 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.getAuth().replayDropRole(privInfo); break; } + case OperationType.OP_CREATE_OUTLINE: { + OutlineInfo outlineInfo = (OutlineInfo) journal.getData(); + env.getOutlineMgr().createOutlineInternal(outlineInfo, true, true); + break; + } + case OperationType.OP_DROP_OUTLINE: { + OutlineInfo outlineInfo = (OutlineInfo) journal.getData(); + env.getOutlineMgr().dropOutlineInternal(outlineInfo.getOutlineName(), true, true); + break; + } case OperationType.OP_UPDATE_USER_PROPERTY: { UserPropertyInfo propertyInfo = (UserPropertyInfo) journal.getData(); env.getAuth().replayUpdateUserProperty(propertyInfo); @@ -1625,6 +1636,14 @@ public void logAlterRole(PrivInfo info) { logEdit(OperationType.OP_ALTER_ROLE, info); } + public void logCreateOutline(OutlineInfo info) { + logEdit(OperationType.OP_CREATE_OUTLINE, info); + } + + public void logDropOutline(OutlineInfo info) { + logEdit(OperationType.OP_DROP_OUTLINE, info); + } + public void logDropRole(PrivInfo info) { logEdit(OperationType.OP_DROP_ROLE, info); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 8636a3dbc5c299..acd8deef26a2e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -415,6 +415,10 @@ public class OperationType { public static final short OP_ALTER_ROLE = 475; + // outline 500 + public static final short OP_CREATE_OUTLINE = 500; + public static final short OP_DROP_OUTLINE = 501; + // For cloud. public static final short OP_UPDATE_CLOUD_REPLICA = 1000; @Deprecated diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java index d2b8737b6a3a38..ec415be0dcdeaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java @@ -260,6 +260,13 @@ public static MetaPersistMethod create(String name) throws NoSuchMethodException metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("savePlsqlProcedure", CountingDataOutputStream.class, long.class); break; + case "outline": + metaPersistMethod.readMethod = + Env.class.getDeclaredMethod("loadOutlines", DataInputStream.class, long.class); + metaPersistMethod.writeMethod = + Env.class.getDeclaredMethod("saveOutlines", CountingDataOutputStream.class, + long.class); + break; default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java index 9f0f0aff0ecff9..eff01e279b2bbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java @@ -42,7 +42,7 @@ public class PersistMetaModules { "paloAuth", "transactionState", "colocateTableIndex", "routineLoadJobs", "loadJobV2", "smallFiles", "plugins", "deleteHandler", "sqlBlockRule", "policy", "globalFunction", "workloadGroups", "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "workloadSchedPolicy", - "insertOverwrite", "plsql"); + "insertOverwrite", "plsql", "outline"); // The modules in `CloudEnv`. public static final ImmutableList CLOUD_MODULE_NAMES = ImmutableList.of("cloudWarmUpJob"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java b/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java index 5da34ae1c2e759..bf1270e09b704b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/OriginStatement.java @@ -38,6 +38,8 @@ public class OriginStatement { @SerializedName(value = "idx") public final int idx; + private String outlineName = null; + public OriginStatement(String originStmt, int idx) { this.originStmt = originStmt; this.idx = idx; @@ -48,6 +50,14 @@ public static OriginStatement read(DataInput in) throws IOException { return GsonUtils.GSON.fromJson(json, OriginStatement.class); } + public void setOutlineName(String outlineName) { + this.outlineName = outlineName; + } + + public String getOutlineName() { + return outlineName; + } + @Override public String toString() { return "OriginStatement{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index aaba0e00f255a5..0b455b2f1ee04b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -337,6 +337,7 @@ public class SessionVariable implements Serializable, Writable { public static final String NTH_OPTIMIZED_PLAN = "nth_optimized_plan"; public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner"; + public static final String ENABLE_SQL_PLAN_OUTLINES = "enable_sql_plan_outlines"; public static final String ENABLE_NEREIDS_DISTRIBUTE_PLANNER = "enable_nereids_distribute_planner"; public static final String DISABLE_NEREIDS_RULES = "disable_nereids_rules"; public static final String ENABLE_NEREIDS_RULES = "enable_nereids_rules"; @@ -1531,6 +1532,9 @@ public boolean isEnableHboNonStrictMatchingMode() { @VariableMgr.VarAttr(name = ENABLE_NEREIDS_PLANNER, needForward = true, varType = VariableAnnotation.REMOVED) private boolean enableNereidsPlanner = true; + @VariableMgr.VarAttr(name = ENABLE_SQL_PLAN_OUTLINES, needForward = true) + private boolean enableSqlPlanOutlines = false; + @VariableMgr.VarAttr(name = DISABLE_NEREIDS_RULES, needForward = true) private String disableNereidsRules = ""; @@ -3793,6 +3797,10 @@ public boolean isEnableNereidsTrace() { return enableNereidsTrace; } + public boolean isEnableSqlPlanOutlines() { + return enableSqlPlanOutlines; + } + public void setEnableExprTrace(boolean enableExprTrace) { this.enableExprTrace = enableExprTrace; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 956b2902d3162b..3378f3d2f8a43d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -407,6 +407,7 @@ private Map getSummaryInfo(boolean isFinished) { builder.defaultDb(context.getDatabase()); builder.workloadGroup(context.getWorkloadGroupName()); builder.sqlStatement(originStmt == null ? "" : originStmt.originStmt); + builder.outlineName(originStmt == null ? "" : originStmt.getOutlineName()); builder.isCached(isCached ? "Yes" : "No"); Map beToInstancesNum = coord == null ? Maps.newTreeMap() : coord.getBeToInstancesNum(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 7b5c717d381a96..7d26391265e3c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -61,6 +61,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.StringJoiner; @@ -831,6 +832,42 @@ public static List> dumpChangedVars(SessionVariable sessionVar) { return changedRows; } + public static List> dumpNereidsVars(SessionVariable sessionVar, HashSet nereidsVars) { + // Hold the read lock when session dump, because this option need to access global variable. + rlock.lock(); + List> changedRows = Lists.newArrayList(); + try { + for (Map.Entry entry : ctxByDisplayVarName.entrySet()) { + VarContext ctx = entry.getValue(); + List row = Lists.newArrayList(); + String varName = entry.getKey(); + String curValue = getValue(sessionVar, ctx.getField()); + String defaultValue = ctx.getDefaultValue(); + if (VariableVarConverters.hasConverter(varName)) { + try { + defaultValue = VariableVarConverters.decode(varName, Long.valueOf(defaultValue)); + curValue = VariableVarConverters.decode(varName, Long.valueOf(curValue)); + } catch (DdlException e) { + LOG.warn("Decode session variable {} failed, reason: {}", varName, e.getMessage()); + } + } + + if (!nereidsVars.contains(varName.toUpperCase())) { + continue; + } + + row.add(varName); + row.add(curValue); + row.add(defaultValue); + changedRows.add(row); + } + } finally { + rlock.unlock(); + } + + return changedRows; + } + @Retention(RetentionPolicy.RUNTIME) public @interface VarAttr { // Name in show variables and set statement; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 0d1afbac39e6de..d2509ec6536309 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -89,6 +89,8 @@ import org.apache.doris.master.MasterImpl; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.hint.OutlineInfo; +import org.apache.doris.nereids.hint.OutlineMgr; import org.apache.doris.nereids.trees.plans.PlanNodeAndHash; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.planner.OlapTableSink; @@ -149,6 +151,8 @@ import org.apache.doris.thrift.TDropPlsqlPackageRequest; import org.apache.doris.thrift.TDropPlsqlStoredProcedureRequest; import org.apache.doris.thrift.TFeResult; +import org.apache.doris.thrift.TFetchOutlineInfoRequest; +import org.apache.doris.thrift.TFetchOutlineInfoResult; import org.apache.doris.thrift.TFetchResourceResult; import org.apache.doris.thrift.TFetchRoutineLoadJobRequest; import org.apache.doris.thrift.TFetchRoutineLoadJobResult; @@ -212,6 +216,7 @@ import org.apache.doris.thrift.TNullableStringLiteral; import org.apache.doris.thrift.TOlapTableIndexTablets; import org.apache.doris.thrift.TOlapTablePartition; +import org.apache.doris.thrift.TOutlineInfo; import org.apache.doris.thrift.TPipelineFragmentParams; import org.apache.doris.thrift.TPipelineWorkloadGroup; import org.apache.doris.thrift.TPlsqlPackageResult; @@ -4310,4 +4315,34 @@ public TFetchRoutineLoadJobResult fetchRoutineLoadJob(TFetchRoutineLoadJobReques return result; } + + @Override + public TFetchOutlineInfoResult fetchOutlineInfo(TFetchOutlineInfoRequest request) { + TFetchOutlineInfoResult result = new TFetchOutlineInfoResult(); + + if (!Env.getCurrentEnv().isReady()) { + return result; + } + + OutlineMgr outlineMgr = Env.getCurrentEnv().getOutlineMgr(); + List outlineInfos = Lists.newArrayList(); + Map outlineInfoMap = outlineMgr.getOutlineMap(); + for (Map.Entry entry : outlineInfoMap.entrySet()) { + TOutlineInfo outlineInfo = new TOutlineInfo(); + OutlineInfo info = entry.getValue(); + outlineInfo.setOutlineName(info.getOutlineName()); + outlineInfo.setVisibleSignature(info.getVisibleSignature()); + outlineInfo.setSqlId(info.getSqlId()); + outlineInfo.setSqlText(info.getSqlText()); + outlineInfo.setOutlineTarget(info.getOutlineTarget()); + outlineInfo.setOutlineData(info.getOutlineData()); + outlineInfos.add(outlineInfo); + } + if (LOG.isDebugEnabled()) { + LOG.debug("outline infos: {}", outlineInfos); + } + result.setOutlineInfos(outlineInfos); + + return result; + } } diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 3a50a36c28d729..54c3143ac9ae84 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -140,7 +140,8 @@ enum TSchemaTableType { SCH_FILE_CACHE_STATISTICS = 51, SCH_CATALOG_META_CACHE_STATISTICS = 52, SCH_BACKEND_KERBEROS_TICKET_CACHE = 53, - SCH_ROUTINE_LOAD_JOBS = 54; + SCH_ROUTINE_LOAD_JOBS = 54, + SCH_OPTIMIZER_SQL_PLAN_OUTLINE = 55; } enum THdfsCompression { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 9e6cddba329664..ac3c187eed16d5 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1608,6 +1608,23 @@ struct TPlanNodeRuntimeStatsItem { 12: optional i32 instance_num } +struct TFetchOutlineInfoRequest { +} + +struct TOutlineInfo { + 1: optional string outline_name + 2: optional string visible_signature + 3: optional string sql_id + 4: optional string sql_text + 5: optional string outline_target + 6: optional string outline_data +} + +struct TFetchOutlineInfoResult { + 1: optional list outlineInfos +} + + service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) @@ -1710,4 +1727,6 @@ service FrontendService { TFetchRunningQueriesResult fetchRunningQueries(1: TFetchRunningQueriesRequest request) TFetchRoutineLoadJobResult fetchRoutineLoadJob(1: TFetchRoutineLoadJobRequest request) + + TFetchOutlineInfoResult fetchOutlineInfo(1: TFetchOutlineInfoRequest request) } diff --git a/regression-test/data/nereids_p0/outline/test_ddl.out b/regression-test/data/nereids_p0/outline/test_ddl.out new file mode 100644 index 00000000000000..321bdbdcf5cc58 --- /dev/null +++ b/regression-test/data/nereids_p0/outline/test_ddl.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +test_outline_ddl_outline1 +test_outline_ddl_outline2 +test_outline_param_outline1 +test_outline_param_outline2 + +-- !sql -- +test_outline_ddl_outline1 + +-- !sql -- +test_outline_ddl_outline2 +test_outline_param_outline1 +test_outline_param_outline2 + diff --git a/regression-test/data/nereids_p0/outline/test_outline_param.out b/regression-test/data/nereids_p0/outline/test_outline_param.out new file mode 100644 index 00000000000000..5db329457c641f --- /dev/null +++ b/regression-test/data/nereids_p0/outline/test_outline_param.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +test_outline_param_outline1 select * from t1 where c1 = ? + +-- !sql -- +test_outline_param_outline2 select * from t1 order by 1 + +-- !sql -- +test_outline_param_outline3 select c1, sum(c1) from t1 group by 1 + diff --git a/regression-test/suites/nereids_p0/outline/test_ddl.groovy b/regression-test/suites/nereids_p0/outline/test_ddl.groovy new file mode 100644 index 00000000000000..54977a71478185 --- /dev/null +++ b/regression-test/suites/nereids_p0/outline/test_ddl.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ddl") { + sql """set enable_sql_plan_outlines=true;""" + String dbName = "test_outline_ddl" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "USE ${dbName}" + String tableName = "t1" + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """create table ${tableName} (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');""" + + sql """drop outline if exists ${dbName}_outline1""" + sql """drop outline if exists ${dbName}_outline2""" + sql """create outline ${dbName}_outline1 on select * from ${tableName}""" + sql """create outline ${dbName}_outline2 on select * from ${tableName} where c1 = 1 and c11 = 2""" + qt_sql """select outline_name from information_schema.optimizer_sql_plan_outline order by outline_name;""" + qt_sql """select outline_name from information_schema.optimizer_sql_plan_outline where outline_name = "${dbName}_outline1" order by outline_name;""" + + // should visible outline1 already exists + test { + sql """create outline ${dbName}_outline1 on select * from ${tableName}""" + exception "${dbName}_outline1 already exists" + } + // should visible outline1 already exists + test { + sql """create outline ${dbName}_outline3 on select * from ${tableName}""" + exception "select * from t1 already exists" + } + // or replace should work + sql """create or replace outline ${dbName}_outline1 on select * from ${tableName}""" + // test drop + sql """drop outline if exists ${dbName}_outline1""" + qt_sql """select outline_name from information_schema.optimizer_sql_plan_outline order by outline_name;""" + sql """create outline ${dbName}_outline1 on select * from ${tableName}""" + // test drop without if exists + sql """drop outline ${dbName}_outline1""" + test { + sql """drop outline ${dbName}_outline1""" + exception "outline1 not exists" + } + sql """drop outline if exists ${dbName}_outline1""" + sql """drop outline if exists ${dbName}_outline2""" + + sql """drop database ${dbName}""" +} diff --git a/regression-test/suites/nereids_p0/outline/test_outline_param.groovy b/regression-test/suites/nereids_p0/outline/test_outline_param.groovy new file mode 100644 index 00000000000000..de2dd036cda28a --- /dev/null +++ b/regression-test/suites/nereids_p0/outline/test_outline_param.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_outline_param") { + sql """set enable_sql_plan_outlines=true;""" + String dbName = "test_outline_param" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "USE ${dbName}" + String tableName = "t1" + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """create table ${tableName} (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');""" + + sql """drop outline if exists ${dbName}_outline1""" + sql """create outline ${dbName}_outline1 on select * from ${tableName} where c1 = 1""" + qt_sql """select outline_name, visible_signature from information_schema.optimizer_sql_plan_outline where outline_name = "${dbName}_outline1";""" + sql """select * from ${tableName} where c1 = 1""" + sql """drop outline if exists ${dbName}_outline2""" + sql """create outline ${dbName}_outline2 on select * from ${tableName} order by 1""" + qt_sql """select outline_name, visible_signature from information_schema.optimizer_sql_plan_outline where outline_name = "${dbName}_outline2";""" + sql """drop outline if exists ${dbName}_outline3""" + sql """create outline ${dbName}_outline3 on select c1, sum(c1) from ${tableName} group by 1""" + qt_sql """select outline_name, visible_signature from information_schema.optimizer_sql_plan_outline where outline_name = "${dbName}_outline3";""" + + sql """drop outline if exists ${dbName}_outline1""" + sql """drop outline if exists ${dbName}_outline2""" + sql """drop outline if exists ${dbName}_outline3""" + + sql """drop database ${dbName}""" +}