From e2f258a580e110ea3986c1614df19f0d236121f2 Mon Sep 17 00:00:00 2001 From: yz Date: Tue, 24 Sep 2024 08:51:59 +0000 Subject: [PATCH] [feature][dingo-executor]Support trace for insert, select limit and order by --- dingo-calcite/src/main/codegen/config.fmpp | 1 + .../src/main/codegen/templates/Parser.jj | 99 ++++++++++++++++++- .../calcite/grammar/dml/SqlInsert.java | 45 +++++++++ .../calcite/grammar/dql/SqlOrderBy.java | 41 ++++++++ .../calcite/grammar/dql/SqlTraceSelect.java | 54 ++++++++++ .../dingodb/common/profile/ExecProfile.java | 3 +- .../dingodb/common/profile/PlanProfile.java | 12 ++- .../io/dingodb/common/profile/Profile.java | 3 +- .../io/dingodb/common/profile/SqlProfile.java | 3 +- .../io/dingodb/driver/DingoDriverParser.java | 9 +- .../java/io/dingodb/driver/DingoMeta.java | 3 +- .../io/dingodb/driver/DingoResultSet.java | 1 - 12 files changed, 263 insertions(+), 11 deletions(-) create mode 100644 dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dml/SqlInsert.java create mode 100644 dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlOrderBy.java create mode 100644 dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlTraceSelect.java diff --git a/dingo-calcite/src/main/codegen/config.fmpp b/dingo-calcite/src/main/codegen/config.fmpp index 2a8a2ed7c0..6fe59c8099 100644 --- a/dingo-calcite/src/main/codegen/config.fmpp +++ b/dingo-calcite/src/main/codegen/config.fmpp @@ -264,6 +264,7 @@ data: { statementParserMethods: [ "SqlTraceSelect" + "SqlTraceInsert" "SqlDescTable" "SqlTruncate" "SqlGrant" diff --git a/dingo-calcite/src/main/codegen/templates/Parser.jj b/dingo-calcite/src/main/codegen/templates/Parser.jj index f723cb6841..6f4636f2ea 100644 --- a/dingo-calcite/src/main/codegen/templates/Parser.jj +++ b/dingo-calcite/src/main/codegen/templates/Parser.jj @@ -731,6 +731,55 @@ SqlNode OrderByLimitOpt(SqlNode e) : } } +/** Reads optional "ORDER BY", "LIMIT", "OFFSET", "FETCH" following a query, + * {@code e}. If any of them are present, adds them to the query; + * otherwise returns the query unchanged. + * Throws if they are present and {@code e} is not a query. */ +SqlNode OrderByLimitTraceOpt(SqlNode e) : +{ + final SqlNodeList orderBy; + final Span s = Span.of(); + SqlNode[] offsetFetch = {null, null}; + ExportOptions exportOptions = null; + boolean forUpdate=false; +} +{ + ( + // use the syntactic type of the expression we just parsed + // to decide whether ORDER BY makes sense + orderBy = OrderBy(e.isA(SqlKind.QUERY)) + | { orderBy = null; } + ) + [ + LimitClause(s, offsetFetch) + [ OffsetClause(s, offsetFetch) ] + | + OffsetClause(s, offsetFetch) + [ + LimitClause(s, offsetFetch) { + if (!this.conformance.isOffsetLimitAllowed()) { + throw SqlUtil.newContextException(s.end(this), + RESOURCE.offsetLimitNotAllowed()); + } + } + | + FetchClause(offsetFetch) + ] + | + FetchClause(offsetFetch) + ] + [ exportOptions = exportOptions() ] + [ { forUpdate = true;} ] + { + if (orderBy != null || offsetFetch[0] != null || offsetFetch[1] != null) { + return new io.dingodb.calcite.grammar.dql.SqlOrderBy(getPos(), e, + Util.first(orderBy, SqlNodeList.EMPTY), + offsetFetch[0], offsetFetch[1], true); + } + return e; + } +} + /** * Parses an OFFSET clause in an ORDER BY expression. */ @@ -1395,8 +1444,10 @@ SqlNode SqlTraceSelect() : final SqlNode having; final SqlNodeList windowDecls; final List hints = new ArrayList(); + final SqlNodeList orderBy; final Span s; boolean forUpdate = false; + SqlNode[] offsetFetch = {null, null}; ExportOptions exportOptions = null; } { @@ -1428,6 +1479,8 @@ SqlNode SqlTraceSelect() : ( groupBy = GroupBy() | { groupBy = null; } ) ( having = Having() | { having = null; } ) ( windowDecls = Window() | { windowDecls = null; } ) + ( orderBy = OrderBy(true) | { orderBy = null; } ) + [ LimitClause(s, offsetFetch) ] [ exportOptions = exportOptions() ] [ { forUpdate=true; }] | @@ -1436,13 +1489,14 @@ SqlNode SqlTraceSelect() : where = null; groupBy = null; having = null; + orderBy = null; windowDecls = null; } ) { return new SqlSelect(s.end(this), keywordList, new SqlNodeList(selectList, Span.of(selectList).pos()), - fromClause, where, groupBy, having, windowDecls, null, null, null, + fromClause, where, groupBy, having, windowDecls, orderBy, null, null, new SqlNodeList(hints, getPos()), exportOptions, true); } @@ -1873,6 +1927,49 @@ SqlNode SqlInsert() : } } +/** + * Parses a leaf insert into expression. + */ +SqlNode SqlTraceInsert() : +{ + final List keywords = new ArrayList(); + final SqlNodeList keywordList; + final SqlIdentifier tableName; + SqlNode tableRef; + SqlNode source; + final SqlNodeList columnList; + final Span s; + final Pair p; +} +{ + + { s = span(); } + SqlInsertKeywords(keywords) { + keywordList = new SqlNodeList(keywords, s.addAll(keywords).pos()); + } + tableName = CompoundTableIdentifier() + ( tableRef = TableHints(tableName) | { tableRef = tableName; } ) + [ LOOKAHEAD(5) tableRef = ExtendTable(tableRef) ] + ( + LOOKAHEAD(2) + p = ParenthesizedCompoundIdentifierList() { + if (p.right.size() > 0) { + tableRef = extend(tableRef, p.right); + } + if (p.left.size() > 0) { + columnList = p.left; + } else { + columnList = null; + } + } + | { columnList = null; } + ) + source = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) { + return new io.dingodb.calcite.grammar.dml.SqlInsert(s.end(source), keywordList, tableRef, source, + columnList, true); + } +} + /* * Abstract production: * diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dml/SqlInsert.java b/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dml/SqlInsert.java new file mode 100644 index 0000000000..b664ea2aa9 --- /dev/null +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dml/SqlInsert.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.calcite.grammar.dml; + +import lombok.Getter; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNumericLiteral; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * add trace + */ +public class SqlInsert extends org.apache.calcite.sql.SqlInsert { + + @Getter + private boolean trace; + + public SqlInsert(SqlParserPos pos, + SqlNodeList keywords, + SqlNode targetTable, + SqlNode source, + @Nullable SqlNodeList columnList, + @Nullable boolean trace) { + super(pos, keywords, targetTable, source, columnList); + this.trace = trace; + } + +} diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlOrderBy.java b/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlOrderBy.java new file mode 100644 index 0000000000..0f8988a520 --- /dev/null +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlOrderBy.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.calcite.grammar.dql; + +import lombok.Getter; +import lombok.Setter; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +@Getter +@Setter +public class SqlOrderBy extends org.apache.calcite.sql.SqlOrderBy { + private boolean trace; + + public SqlOrderBy(SqlParserPos pos, + SqlNode query, + SqlNodeList orderList, + @Nullable SqlNode offset, + @Nullable SqlNode fetch, + @Nullable boolean trace) { + super(pos, query, orderList, offset, fetch); + this.trace = trace; + } + +} diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlTraceSelect.java b/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlTraceSelect.java new file mode 100644 index 0000000000..b32f945200 --- /dev/null +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/grammar/dql/SqlTraceSelect.java @@ -0,0 +1,54 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.calcite.grammar.dql; + +import lombok.Getter; +import lombok.Setter; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.UUID; + +@Getter +public class SqlTraceSelect extends SqlSelect { + + @Setter + ExportOptions exportOptions; + + boolean trace; + + public SqlTraceSelect(SqlParserPos pos, + @Nullable SqlNodeList keywordList, + SqlNodeList selectList, + @Nullable SqlNode from, + @Nullable SqlNode where, + @Nullable SqlNodeList groupBy, + @Nullable SqlNode having, + @Nullable SqlNodeList windowDecls, + @Nullable SqlNodeList orderBy, + @Nullable SqlNode offset, + @Nullable SqlNode fetch, + @Nullable SqlNodeList hints, + ExportOptions exportOptions, + boolean trace) { + super(pos, keywordList, selectList, from, where, groupBy, having, windowDecls, orderBy, offset, fetch, hints); + this.exportOptions = exportOptions; + this.trace = trace; + } +} diff --git a/dingo-common/src/main/java/io/dingodb/common/profile/ExecProfile.java b/dingo-common/src/main/java/io/dingodb/common/profile/ExecProfile.java index 05fe86cf58..12a308484a 100644 --- a/dingo-common/src/main/java/io/dingodb/common/profile/ExecProfile.java +++ b/dingo-common/src/main/java/io/dingodb/common/profile/ExecProfile.java @@ -63,10 +63,11 @@ public void traceTree(byte[] prefix, List rowList) { } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } - Object[] val = new Object[3]; + Object[] val = new Object[4]; val[0] = prefixStr + "runStmt"; val[1] = DateTimeUtils.timeFormat(new Time(start)); val[2] = String.valueOf(duration); + val[3] = Long.valueOf(this.getCount()); rowList.add(val); if (profile != null) { diff --git a/dingo-common/src/main/java/io/dingodb/common/profile/PlanProfile.java b/dingo-common/src/main/java/io/dingodb/common/profile/PlanProfile.java index 0bb176c5fa..301c079876 100644 --- a/dingo-common/src/main/java/io/dingodb/common/profile/PlanProfile.java +++ b/dingo-common/src/main/java/io/dingodb/common/profile/PlanProfile.java @@ -71,10 +71,11 @@ public void traceTree(byte[] prefix, List rowList) { } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } - Object[] val = new Object[3]; + Object[] val = new Object[4]; val[0] = termStr + "compile"; val[1] = DateTimeUtils.timeFormat(new Time(start)); val[2] = String.valueOf(duration); + val[3] = Long.valueOf(this.getCount()); rowList.add(val); byte[] prefix1 = new byte[prefix.length + 2]; @@ -85,22 +86,25 @@ public void traceTree(byte[] prefix, List rowList) { } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } - Object[] parseVal = new Object[3]; + Object[] parseVal = new Object[4]; parseVal[0] = termStr + "parse"; parseVal[1] = DateTimeUtils.timeFormat(new Time(start)); parseVal[2] = String.valueOf(parse); + parseVal[3] = Long.valueOf(this.getCount()); rowList.add(parseVal); - Object[] validateVal = new Object[3]; + Object[] validateVal = new Object[4]; validateVal[0] = termStr + "validate"; validateVal[1] = DateTimeUtils.timeFormat(new Time(parseTime)); validateVal[2] = String.valueOf(validate); + validateVal[3] = Long.valueOf(this.getCount()); rowList.add(validateVal); - Object[] optimizeVal = new Object[3]; + Object[] optimizeVal = new Object[4]; optimizeVal[0] = termStr + "optimize"; optimizeVal[1] = DateTimeUtils.timeFormat(new Time(validateTime)); optimizeVal[2] = String.valueOf(optimize); + optimizeVal[3] = Long.valueOf(this.getCount()); rowList.add(optimizeVal); } diff --git a/dingo-common/src/main/java/io/dingodb/common/profile/Profile.java b/dingo-common/src/main/java/io/dingodb/common/profile/Profile.java index e118f21a1a..11b2b0338e 100644 --- a/dingo-common/src/main/java/io/dingodb/common/profile/Profile.java +++ b/dingo-common/src/main/java/io/dingodb/common/profile/Profile.java @@ -107,7 +107,7 @@ public void traceTree(Profile profile, byte[] prefix, List rowList) { && !"source".equals(profile.type) && !"root".equals(profile.type) && profile.getEnd() > 0 && profile.getStart() > 0 ) { - Object[] val = new Object[3]; + Object[] val = new Object[4]; val[0] = node + profile.type; if (profile instanceof SourceProfile) { SourceProfile sourceProfile = (SourceProfile) profile; @@ -118,6 +118,7 @@ public void traceTree(Profile profile, byte[] prefix, List rowList) { } val[1] = DateTimeUtils.timeFormat(new Time(profile.start)); val[2] = String.valueOf(profile.getDuration()); + val[3] = Long.valueOf(this.getCount()); rowList.add(val); skip = false; } diff --git a/dingo-common/src/main/java/io/dingodb/common/profile/SqlProfile.java b/dingo-common/src/main/java/io/dingodb/common/profile/SqlProfile.java index b477ed07f3..218305f463 100644 --- a/dingo-common/src/main/java/io/dingodb/common/profile/SqlProfile.java +++ b/dingo-common/src/main/java/io/dingodb/common/profile/SqlProfile.java @@ -101,10 +101,11 @@ public List traceTree(List rowList) { } if (execProfile != null) { if (planEndTime > 0) { - Object[] val = new Object[3]; + Object[] val = new Object[4]; val[0] = termStr + "schedule job"; val[1] = DateTimeUtils.timeFormat(new Time(execProfile.getStart())); val[2] = String.valueOf(execProfile.start - planEndTime); + val[3] = Long.valueOf(this.getCount()); rowList.add(val); } execProfile.traceTree(prefix, rowList); diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java index 6581cfd9a5..f58c2ae40f 100644 --- a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java +++ b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java @@ -443,6 +443,7 @@ public Meta.Signature parseQuery( DingoMetrics.timer("relOptimize").update(sub, TimeUnit.MILLISECONDS); planProfile.endOptimize(); markAutoIncForDml(relNode); + Location currentLocation = MetaService.root().currentLocation(); RelDataType parasType = validator.getParameterRowType(sqlNode); Set tables = useTables(relNode, sqlNode); @@ -977,6 +978,8 @@ public void deepSugar(List sqlNodes) { private static boolean trace(SqlNode sqlNode) { if (sqlNode instanceof io.dingodb.calcite.grammar.dql.SqlSelect) { return ((io.dingodb.calcite.grammar.dql.SqlSelect) sqlNode).isTrace(); + } else if (sqlNode instanceof io.dingodb.calcite.grammar.dml.SqlInsert) { + return ((io.dingodb.calcite.grammar.dml.SqlInsert) sqlNode).isTrace(); } return false; } @@ -986,12 +989,16 @@ private static List getTraceColMeta(JavaTypeFactory typeFactory) new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.CHAR), null, false); ColumnMetaData colMeta2 = metaData(typeFactory, 1, "startTs", new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.CHAR), null, false); - ColumnMetaData colMeta3 = metaData(typeFactory, 1, "duration", + ColumnMetaData colMeta3 = metaData(typeFactory, 2, "duration", new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.CHAR), null, false); + ColumnMetaData colMeta4 = metaData(typeFactory, 3, "rowcount", + new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.BIGINT), null, true); + List metaDataList = new ArrayList<>(); metaDataList.add(colMeta1); metaDataList.add(colMeta2); metaDataList.add(colMeta3); + metaDataList.add(colMeta4); return metaDataList; } diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoMeta.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoMeta.java index b33721e55e..3dacc780b8 100644 --- a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoMeta.java +++ b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoMeta.java @@ -323,6 +323,7 @@ public ExecuteResult prepareAndExecute( } // For local driver, here `fetch` is called. callback.execute(); + if (signature.statementType == StatementType.OTHER_DDL) { addSqlProfile(statement.getSqlProfile(), connection); ((DingoConnection) connection).setCommandStartTime(0); @@ -1363,7 +1364,7 @@ private static void initProfile(SqlProfile sqlProfile, DingoConnection dingoConn public static void getTraceValues(SqlProfile sqlProfile, List rowList) { long duration = System.currentTimeMillis() - sqlProfile.getStart(); String startTs = DateTimeUtils.timeFormat(new Time(sqlProfile.getStart())); - rowList.add(new Object[] {"trace", startTs, String.valueOf(duration)}); + rowList.add(new Object[] {"trace", startTs, String.valueOf(duration), Long.valueOf(0)}); sqlProfile.traceTree(rowList); } } diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSet.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSet.java index 034ccf6dde..be2f8ad238 100644 --- a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSet.java +++ b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSet.java @@ -16,7 +16,6 @@ package io.dingodb.driver; -import io.dingodb.exec.base.JobIterator; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j;