diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexFullScanVisitFun.java b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexFullScanVisitFun.java index a48ef9eab9..76631f35f3 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexFullScanVisitFun.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexFullScanVisitFun.java @@ -148,7 +148,8 @@ private DingoIndexFullScanVisitFun() { relOp, rel.isPushDown(), rel.getSelection(), - 0 + 0, + transaction.isAutoCommit() )); } assert indexScanvertex != null; diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexRangeScanVisitFun.java b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexRangeScanVisitFun.java index e19430d6fd..020e543cc2 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexRangeScanVisitFun.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexRangeScanVisitFun.java @@ -175,7 +175,8 @@ private DingoIndexRangeScanVisitFun() { relOp, rel.isPushDown(), rel.getSelection(), - 0 + 0, + transaction.isAutoCommit() )); } OutputHint hint = new OutputHint(); diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexScanWithRelOpVisitFun.java b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexScanWithRelOpVisitFun.java index 9d76caeccd..9d198ce737 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexScanWithRelOpVisitFun.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoIndexScanWithRelOpVisitFun.java @@ -188,7 +188,8 @@ private DingoIndexScanWithRelOpVisitFun() { rel.isPushDown(), td.version, 0, - td.getCodecVersion() + td.getCodecVersion(), + null ); if (relOp instanceof PipeOp) { return new Vertex(SCAN_WITH_PIPE_OP, param); @@ -231,7 +232,9 @@ private DingoIndexScanWithRelOpVisitFun() { rel.isPushDown(), td.version, 0, - td.getCodecVersion() + td.getCodecVersion(), + null, + transaction.isAutoCommit() ); if (relOp instanceof PipeOp) { return new Vertex(TXN_SCAN_WITH_PIPE_OP, param); diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoScanWithRelOpVisitFun.java b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoScanWithRelOpVisitFun.java index 00a3f35520..251d104769 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoScanWithRelOpVisitFun.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/function/DingoScanWithRelOpVisitFun.java @@ -212,7 +212,8 @@ private DingoScanWithRelOpVisitFun() { rel.isPushDown(), td.version, rel.getLimit(), - td.getCodecVersion() + td.getCodecVersion(), + null ); if (relOp instanceof PipeOp) { return new Vertex(SCAN_WITH_PIPE_OP, param); @@ -256,7 +257,9 @@ private DingoScanWithRelOpVisitFun() { rel.isPushDown(), td.version, rel.getLimit(), - td.getCodecVersion() + td.getCodecVersion(), + null, + transaction.isAutoCommit() ); if (relOp instanceof PipeOp) { return new Vertex(TXN_SCAN_WITH_PIPE_OP, param); diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/params/ScanWithRelOpParam.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/params/ScanWithRelOpParam.java index d1feee3d91..fa986045ec 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/operator/params/ScanWithRelOpParam.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/operator/params/ScanWithRelOpParam.java @@ -42,6 +42,7 @@ import io.dingodb.expr.runtime.expr.NullaryAggExpr; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.NonNull; import java.io.ByteArrayOutputStream; @@ -52,6 +53,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +@Slf4j @JsonTypeName("scanRel") @JsonPropertyOrder({ "tableId", @@ -79,6 +81,9 @@ public class ScanWithRelOpParam extends ScanParam { @Setter protected int limit; + @JsonProperty("selection") + protected List selection; + @Getter protected transient CoprocessorV2 coprocessor; @@ -126,7 +131,8 @@ public ScanWithRelOpParam( boolean pushDown, int schemaVersion, int limit, - int codecVersion + int codecVersion, + List selection ) { super(tableId, schema, keyMapping, schemaVersion, codecVersion); this.relOp = relOp; @@ -135,6 +141,7 @@ public ScanWithRelOpParam( coprocessor = null; this.limit = limit; config = new DingoRelConfig(); + this.selection = selection; } @Override diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnIndexRangeScanParam.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnIndexRangeScanParam.java index 034741f7fa..dab7f7cc63 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnIndexRangeScanParam.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnIndexRangeScanParam.java @@ -23,24 +23,34 @@ import io.dingodb.codec.KeyValueCodec; import io.dingodb.common.CommonId; import io.dingodb.common.CoprocessorV2; +import io.dingodb.common.log.LogUtils; import io.dingodb.common.type.DingoType; import io.dingodb.common.type.TupleMapping; import io.dingodb.exec.dag.Vertex; import io.dingodb.exec.expr.DingoCompileContext; import io.dingodb.exec.utils.SchemaWrapperUtils; +import io.dingodb.exec.utils.relop.RelOpMappingVisitor; +import io.dingodb.exec.utils.relop.RelOpSelectionVisitor; +import io.dingodb.exec.utils.relop.SelectionFlag; +import io.dingodb.exec.utils.relop.SelectionObj; import io.dingodb.expr.coding.CodingFlag; import io.dingodb.expr.coding.RelOpCoder; import io.dingodb.expr.common.type.TupleType; import io.dingodb.expr.rel.RelOp; import io.dingodb.meta.entity.Table; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; +@Slf4j @Getter public class TxnIndexRangeScanParam extends ScanWithRelOpParam { @@ -68,6 +78,8 @@ public class TxnIndexRangeScanParam extends ScanWithRelOpParam { protected List mapList; @JsonProperty("selection") private TupleMapping selection; + @JsonProperty("isAutoCommit") + private final boolean isAutoCommit; public TxnIndexRangeScanParam(CommonId indexTableId, CommonId tableId, @@ -82,9 +94,10 @@ public TxnIndexRangeScanParam(CommonId indexTableId, RelOp relOp, boolean pushDown, TupleMapping selection, - int limit) { + int limit, + boolean isAutoCommit) { super(tableId, index.tupleType(), keyMapping, relOp, outputSchema, - pushDown, index.getVersion(), limit, table.getCodecVersion()); + pushDown, index.getVersion(), limit, table.getCodecVersion(), selection.stream().boxed().collect(Collectors.toList())); this.indexSchema = index.tupleType(); this.indexTableId = indexTableId; this.isLookup = isLookup; @@ -94,6 +107,7 @@ public TxnIndexRangeScanParam(CommonId indexTableId, this.scanTs = scanTs; this.timeout = timeout; this.selection = selection; + this.isAutoCommit = isAutoCommit; this.codec = CodecService.getDefault().createKeyValueCodec( index.getCodecVersion(), index.version, index.tupleType(), index.keyMapping()); if (isLookup) { @@ -109,16 +123,48 @@ public void init(Vertex vertex) { if (relOp == null) { return; } - relOp = relOp.compile(new DingoCompileContext( + RelOp relOpCompile = relOp.compile(new DingoCompileContext( (TupleType) indexSchema.getType(), (TupleType) vertex.getParasType().getType() ), config); if (pushDown) { ByteArrayOutputStream os = new ByteArrayOutputStream(); - if (RelOpCoder.INSTANCE.visit(relOp, os) == CodingFlag.OK) { + if (RelOpCoder.INSTANCE.visit(relOpCompile, os) == CodingFlag.OK) { List selection = IntStream.range(0, indexSchema.fieldCount()) .boxed() .collect(Collectors.toList()); + Set selections = new HashSet<>(); + SelectionObj selectionObj = new SelectionObj(selections, true); + boolean isSelection = false; + if (isAutoCommit() && RelOpSelectionVisitor.INSTANCE.visit(relOp, selectionObj) == SelectionFlag.OK + && selectionObj.isProject() && selections.size() != selection.size()) { + try { + selection.clear(); + selection.addAll(selections); + selection.sort(Comparator.naturalOrder()); + relOpCompile = RelOpMappingVisitor.INSTANCE.visit(relOp, selection); + LogUtils.debug(log, "jobId:{}, new relOp: {}", vertex.getTask().getJobId(), relOpCompile); + isSelection = true; + } catch (Exception e) { + LogUtils.error(log, e.getMessage(), e); + selection = IntStream.range(0, indexSchema.fieldCount()) + .boxed() + .collect(Collectors.toList()); + } + } else { + LogUtils.debug(log, "jobId:{}, origin relOp: {}", vertex.getTask().getJobId(), relOp); + } + if (isSelection) { + relOpCompile = relOpCompile.compile(new DingoCompileContext( + (TupleType) indexSchema.select(TupleMapping.of(selection)).getType(), + (TupleType) vertex.getParasType().getType() + ), config); + os = new ByteArrayOutputStream(); + if (RelOpCoder.INSTANCE.visit(relOpCompile, os) != CodingFlag.OK) { + relOp = relOpCompile; + return; + } + } TupleMapping keyMapping = indexKeyMapping(); TupleMapping outputKeyMapping = TupleMapping.of(new int[]{}); coprocessor = CoprocessorV2.builder() @@ -130,6 +176,7 @@ public void init(Vertex vertex) { .build(); } } + relOp = relOpCompile; } public TupleMapping indexKeyMapping() { diff --git a/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnScanWithRelOpParam.java b/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnScanWithRelOpParam.java index 6529e64228..3fe086e03b 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnScanWithRelOpParam.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/operator/params/TxnScanWithRelOpParam.java @@ -20,12 +20,34 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonTypeName; import io.dingodb.common.CommonId; +import io.dingodb.common.CoprocessorV2; +import io.dingodb.common.log.LogUtils; import io.dingodb.common.type.DingoType; import io.dingodb.common.type.TupleMapping; +import io.dingodb.exec.dag.Vertex; +import io.dingodb.exec.expr.DingoCompileContext; +import io.dingodb.exec.utils.SchemaWrapperUtils; +import io.dingodb.exec.utils.relop.RelOpMappingVisitor; +import io.dingodb.exec.utils.relop.RelOpSelectionVisitor; +import io.dingodb.exec.utils.relop.SelectionFlag; +import io.dingodb.exec.utils.relop.SelectionObj; +import io.dingodb.expr.coding.CodingFlag; +import io.dingodb.expr.coding.RelOpCoder; +import io.dingodb.expr.common.type.TupleType; import io.dingodb.expr.rel.RelOp; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.NonNull; +import java.io.ByteArrayOutputStream; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Slf4j @Getter @JsonTypeName("txnScanRel") @JsonPropertyOrder({ @@ -46,6 +68,7 @@ public class TxnScanWithRelOpParam extends ScanWithRelOpParam { private final long timeOut; @JsonProperty("scanTs") private long scanTs; + private final boolean isAutoCommit; public TxnScanWithRelOpParam( CommonId tableId, @@ -59,12 +82,15 @@ public TxnScanWithRelOpParam( boolean pushDown, int schemaVersion, int limit, - int codecVersion + int codecVersion, + List selection, + boolean isAutoCommit ) { - super(tableId, schema, keyMapping, relOp, outputSchema, pushDown, schemaVersion, limit, codecVersion); + super(tableId, schema, keyMapping, relOp, outputSchema, pushDown, schemaVersion, limit, codecVersion, selection); this.scanTs = scanTs; this.isolationLevel = isolationLevel; this.timeOut = timeOut; + this.isAutoCommit = isAutoCommit; } @Override @@ -72,4 +98,67 @@ public void setStartTs(long startTs) { super.setStartTs(startTs); this.scanTs = startTs; } + + @Override + public void init(Vertex vertex) { + if (selection == null) { + selection = IntStream.range(0, schema.fieldCount()) + .boxed() + .collect(Collectors.toList()); + } + RelOp relOpCompile = relOp.compile(new DingoCompileContext( + (TupleType) schema.getType(), + (TupleType) vertex.getParasType().getType() + ), config); + if (pushDown) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + if (RelOpCoder.INSTANCE.visit(relOpCompile, os) == CodingFlag.OK) { + Set selections = new HashSet<>(); + SelectionObj selectionObj = new SelectionObj(selections, true); + boolean isSelection = false; + if (isAutoCommit() && RelOpSelectionVisitor.INSTANCE.visit(relOp, selectionObj) == SelectionFlag.OK + && selectionObj.isProject() && selections.size() != selection.size()) { + try { + selection.clear(); + selection.addAll(selections); + selection.sort(Comparator.naturalOrder()); + relOpCompile = RelOpMappingVisitor.INSTANCE.visit(relOp, selection); + LogUtils.debug(log, "jobId:{}, new relOp: {}", vertex.getTask().getJobId(), relOpCompile); + isSelection = true; + } catch (Exception e) { + LogUtils.error(log, e.getMessage(), e); + selection = IntStream.range(0, schema.fieldCount()) + .boxed() + .collect(Collectors.toList()); + } + } else { + LogUtils.debug(log, "jobId:{}, origin relOp: {}", vertex.getTask().getJobId(), relOp); + } + if (isSelection) { + relOpCompile = relOpCompile.compile(new DingoCompileContext( + (TupleType) schema.select(TupleMapping.of(selection)).getType(), + (TupleType) vertex.getParasType().getType() + ), config); + os = new ByteArrayOutputStream(); + if (RelOpCoder.INSTANCE.visit(relOpCompile, os) != CodingFlag.OK) { + relOp = relOpCompile; + return; + } + } + TupleMapping outputKeyMapping = TupleMapping.of(new int[]{}); + coprocessor = CoprocessorV2.builder() + .originalSchema(SchemaWrapperUtils.buildSchemaWrapper(schema, keyMapping, tableId.seq)) + .resultSchema(SchemaWrapperUtils.buildSchemaWrapper(outputSchema, outputKeyMapping, tableId.seq)) + .selection(selection) + .relExpr(os.toByteArray()) + .codecVersion(codecVersion) + .build(); + if (limit > 0) { + coprocessor.setLimit(limit); + } + } + } + relOp = relOpCompile; + } + } diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ExprMappingVisitor.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ExprMappingVisitor.java new file mode 100644 index 0000000000..a3da27bf50 --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ExprMappingVisitor.java @@ -0,0 +1,142 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import io.dingodb.expr.common.type.IntType; +import io.dingodb.expr.runtime.expr.BinaryOpExpr; +import io.dingodb.expr.runtime.expr.Expr; +import io.dingodb.expr.runtime.expr.ExprVisitorBase; +import io.dingodb.expr.runtime.expr.Exprs; +import io.dingodb.expr.runtime.expr.IndexOpExpr; +import io.dingodb.expr.runtime.expr.NullaryAggExpr; +import io.dingodb.expr.runtime.expr.NullaryOpExpr; +import io.dingodb.expr.runtime.expr.TertiaryOpExpr; +import io.dingodb.expr.runtime.expr.UnaryAggExpr; +import io.dingodb.expr.runtime.expr.UnaryOpExpr; +import io.dingodb.expr.runtime.expr.Val; +import io.dingodb.expr.runtime.expr.Var; +import io.dingodb.expr.runtime.expr.VariadicOpExpr; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.NonNull; + +import java.util.List; + +import static io.dingodb.exec.utils.relop.SelectionMapping.findIndex; + + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class ExprMappingVisitor extends ExprVisitorBase> { + public static final ExprMappingVisitor INSTANCE = new ExprMappingVisitor(); + + +// private boolean cascadingBinaryLogical(Set selected, Expr @NonNull [] operands) throws IOException { +// if (visit(operands[0], selected) != SelectionFlag.OK) { +// return false; +// } +// for (int i = 1; i < operands.length; ++i) { +// if (visit(operands[i], selected) != SelectionFlag.OK) { +// return false; +// } +//// selected.add(operands.) +// } +// return true; +// } + + @Override + public Expr visitVal(@NonNull Val expr, @NonNull List selection) { + return new ValMappingVisitor(expr).visit(expr.getType(), selection); + } + + @SneakyThrows + @Override + public Expr visitVar(@NonNull Var expr, List selection) { + return Exprs.var(expr.getId()); + } + + @Override + public Expr visitNullaryOpExpr(@NonNull NullaryOpExpr expr, @NonNull List selection) { + return Exprs.op(expr.getOp()); + } + + @SneakyThrows + @Override + public Expr visitUnaryOpExpr(@NonNull UnaryOpExpr expr, @NonNull List selection) { + Expr expr1 = visit(expr.getOperand(), selection); + return Exprs.op(expr.getOp(), expr1); + } + + @SneakyThrows + @Override + public Expr visitBinaryOpExpr(@NonNull BinaryOpExpr expr, @NonNull List selection) { + Expr expr0 = visit(expr.getOperand0(), selection); + Expr expr1 = visit(expr.getOperand1(), selection); + return Exprs.op(expr.getOp(), expr0, expr1); + } + + @SneakyThrows + @Override + public Expr visitTertiaryOpExpr(@NonNull TertiaryOpExpr expr, @NonNull List selection) { + Expr expr0 = visit(expr.getOperand0(), selection); + Expr expr1 = visit(expr.getOperand1(), selection); + Expr expr2 = visit(expr.getOperand2(), selection); + return Exprs.op(expr.getOp(), expr0, expr1, expr2); + } + + @SneakyThrows + @Override + public Expr visitVariadicOpExpr(@NonNull VariadicOpExpr expr, @NonNull List selection) { + + Expr[] operands = expr.getOperands(); + Object[] newOperands = new Expr[operands.length]; + for (int i = 0; i < operands.length; i++) { + Expr expr1 = visit(operands[i], selection); + newOperands[i] = expr1; + } + return Exprs.op(expr.getOp(), newOperands); + } + + @Override + public Expr visitIndexOpExpr(@NonNull IndexOpExpr expr, @NonNull List selection) { + Expr expr0 = visit(expr.getOperand0(), selection); + if (expr.getOperand1() instanceof Val && expr.getOperand1().getType() instanceof IntType) { + Integer value = (Integer) ((Val) expr.getOperand1()).getValue(); + if (value != null) { + value = findIndex(selection, value); + } + return Exprs.op(Exprs.INDEX, expr0, Exprs.val(value, expr.getOperand1().getType())); + } + Expr expr1 = visit(expr.getOperand1(), selection); + return Exprs.op(Exprs.INDEX, expr0, expr1); + } + + @SneakyThrows + @Override + public Expr visitNullaryAggExpr(@NonNull NullaryAggExpr expr, @NonNull List selection) { + return Exprs.op(expr.getOp()); + } + + @SneakyThrows + @Override + public Expr visitUnaryAggExpr(@NonNull UnaryAggExpr expr, @NonNull List selection) { + Expr operand = visit(expr.getOperand(), selection); + return Exprs.op(expr.getOp(), operand); + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ExprSelectionVisitor.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ExprSelectionVisitor.java new file mode 100644 index 0000000000..dc824c166e --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ExprSelectionVisitor.java @@ -0,0 +1,208 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import io.dingodb.common.log.LogUtils; +import io.dingodb.expr.common.type.IntType; +import io.dingodb.expr.runtime.expr.BinaryOpExpr; +import io.dingodb.expr.runtime.expr.Expr; +import io.dingodb.expr.runtime.expr.ExprVisitorBase; +import io.dingodb.expr.runtime.expr.IndexOpExpr; +import io.dingodb.expr.runtime.expr.NullaryAggExpr; +import io.dingodb.expr.runtime.expr.NullaryOpExpr; +import io.dingodb.expr.runtime.expr.TertiaryOpExpr; +import io.dingodb.expr.runtime.expr.UnaryAggExpr; +import io.dingodb.expr.runtime.expr.UnaryOpExpr; +import io.dingodb.expr.runtime.expr.Val; +import io.dingodb.expr.runtime.expr.Var; +import io.dingodb.expr.runtime.expr.VariadicOpExpr; +import io.dingodb.expr.runtime.op.OpType; +import io.dingodb.expr.runtime.op.aggregation.CountAllAgg; +import io.dingodb.expr.runtime.op.logical.AndFun; +import io.dingodb.expr.runtime.op.logical.OrFun; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.NonNull; + +import java.io.IOException; +import java.util.Set; + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class ExprSelectionVisitor extends ExprVisitorBase> { + public static final ExprSelectionVisitor INSTANCE = new ExprSelectionVisitor(); + + + private boolean cascadingBinaryLogical(Set selected, Expr @NonNull [] operands) throws IOException { + if (visit(operands[0], selected) != SelectionFlag.OK) { + return false; + } + for (int i = 1; i < operands.length; ++i) { + if (visit(operands[i], selected) != SelectionFlag.OK) { + return false; + } +// selected.add(operands.) + } + return true; + } + + @Override + public SelectionFlag visitVal(@NonNull Val expr, Set selected) { + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitVar(@NonNull Var expr, Set selected) { + return SelectionFlag.OK; + } + + @Override + public SelectionFlag visitNullaryOpExpr(@NonNull NullaryOpExpr expr, @NonNull Set selected) { + return super.visitNullaryOpExpr(expr, selected); + } + + @SneakyThrows + @Override + public SelectionFlag visitUnaryOpExpr(@NonNull UnaryOpExpr expr, Set selected) { + if (visit(expr.getOperand(), selected) == SelectionFlag.OK) { + boolean success = false; + switch (expr.getOpType()) { + case POS: + case NEG: + case NOT: + case CAST: + success = true; + break; + case FUN: + switch (expr.getOp().getName()) { + default: + success = true; + break; + } + break; + default: + break; + } + if (success) { + return SelectionFlag.OK; + } + } + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitBinaryOpExpr(@NonNull BinaryOpExpr expr, Set selected) { + if (visit(expr.getOperand0(), selected) == SelectionFlag.OK && visit(expr.getOperand1(), selected) == SelectionFlag.OK) { + boolean success = false; + switch (expr.getOpType()) { + case ADD: + case SUB: + case MUL: + case DIV: + case EQ: + case NE: + case GT: + case GE: + case LT: + case LE: + case AND: + case OR: + success = true; + break; + case FUN: + switch (expr.getOp().getName()) { + default: + success = true; + break; + } + break; + default: + break; + } + if (success) { + return SelectionFlag.OK; + } + } + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitTertiaryOpExpr(@NonNull TertiaryOpExpr expr, @NonNull Set selected) { + if (visit(expr.getOperand0(), selected) == SelectionFlag.OK + && visit(expr.getOperand1(), selected) == SelectionFlag.OK + && visit(expr.getOperand2(), selected) == SelectionFlag.OK + ) { + boolean success = false; + if (expr.getOpType() == OpType.FUN) { + success = true; + } + if (success) { + return SelectionFlag.OK; + } + } + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitVariadicOpExpr(@NonNull VariadicOpExpr expr, Set selected) { + if (expr.getOpType() == OpType.FUN) { + boolean success = false; + switch (expr.getOp().getName()) { + case AndFun.NAME: + case OrFun.NAME: + success = true; + break; + default: + break; + } + if (success) { + return SelectionFlag.OK; + } + } + return null; + } + + @Override + public SelectionFlag visitIndexOpExpr(@NonNull IndexOpExpr expr, Set selected) { + if (expr.getOperand1() instanceof Val && expr.getOperand1().getType() instanceof IntType) { + Integer value = (Integer) ((Val) expr.getOperand1()).getValue(); + selected.add(value); + } + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitNullaryAggExpr(@NonNull NullaryAggExpr expr, @NonNull Set selected) { + if (expr.getOp().getName().equals(CountAllAgg.NAME)) { + return SelectionFlag.OK; + } + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitUnaryAggExpr(@NonNull UnaryAggExpr expr, @NonNull Set selected) { + return SelectionFlag.OK; + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/RelOpMappingVisitor.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/RelOpMappingVisitor.java new file mode 100644 index 0000000000..c21a13ab34 --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/RelOpMappingVisitor.java @@ -0,0 +1,103 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import io.dingodb.expr.rel.RelOp; +import io.dingodb.expr.rel.RelOpVisitorBase; +import io.dingodb.expr.rel.SourceOp; +import io.dingodb.expr.rel.TandemOp; +import io.dingodb.expr.rel.op.FilterOp; +import io.dingodb.expr.rel.op.GroupedAggregateOp; +import io.dingodb.expr.rel.op.ProjectOp; +import io.dingodb.expr.rel.op.RelOpBuilder; +import io.dingodb.expr.rel.op.UngroupedAggregateOp; +import io.dingodb.expr.runtime.expr.Expr; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.checkerframework.checker.nullness.qual.NonNull; + +import java.util.List; + + +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class RelOpMappingVisitor extends RelOpVisitorBase> { + public static final RelOpMappingVisitor INSTANCE = new RelOpMappingVisitor(); + + private static final ExprMappingVisitor EXPR_MAPPING_VISITOR = ExprMappingVisitor.INSTANCE; + +// @SneakyThrows +// private static @Nullable List visitAggList(@NonNull List aggList, @NonNull RelOp relOp) { +// for (Expr expr : aggList) { +// if (EXPR_SELECTION.visit(expr, selected) != SelectionFlag.OK) { +// return null; +// } +// } +// return SelectionFlag.OK; +// } + + @Override + public RelOp visitSourceOp(SourceOp op, @NonNull List selection) { + return null; + } + + @SneakyThrows + @Override + public RelOp visitFilterOp(@NonNull FilterOp op, @NonNull List selection) { + Expr expr1 = EXPR_MAPPING_VISITOR.visit(op.getFilter(), selection); + RelOp relOp = RelOpBuilder.builder() + .filter(expr1) + .build(); + return relOp; + } + + @SneakyThrows + @Override + public RelOp visitProjectOp(@NonNull ProjectOp op, @NonNull List selection) { + Expr[] newProjects = new Expr[op.getProjects().length]; + int i = 0; + for (Expr expr : op.getProjects()) { + Expr expr1 = EXPR_MAPPING_VISITOR.visit(expr, selection); + newProjects[i] = expr1; + i++; + } + RelOp relOp = RelOpBuilder.builder() + .project(newProjects) + .build(); + return relOp; + } + + @Override + public RelOp visitTandemOp(@NonNull TandemOp op, @NonNull List selection) { + RelOp op1 = visit(op.getInput(), selection); + RelOp op2 = visit(op.getOutput(), selection); + RelOp relOp = RelOpBuilder.builder(op1).add(op2).build(); + return relOp; + } + + @SneakyThrows + @Override + public RelOp visitUngroupedAggregateOp(@NonNull UngroupedAggregateOp op, @NonNull List selection) { + return RelOpBuilder.builder().agg(null, op.getAggList()).build(); + } + + @SneakyThrows + @Override + public RelOp visitGroupedAggregateOp(@NonNull GroupedAggregateOp op, @NonNull List selection) { + return RelOpBuilder.builder().agg(op.getGroupIndices(), op.getAggList()).build(); + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/RelOpSelectionVisitor.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/RelOpSelectionVisitor.java new file mode 100644 index 0000000000..ffe9fe2509 --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/RelOpSelectionVisitor.java @@ -0,0 +1,100 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import io.dingodb.expr.rel.RelOpVisitorBase; +import io.dingodb.expr.rel.SourceOp; +import io.dingodb.expr.rel.TandemOp; +import io.dingodb.expr.rel.op.FilterOp; +import io.dingodb.expr.rel.op.GroupedAggregateOp; +import io.dingodb.expr.rel.op.ProjectOp; +import io.dingodb.expr.rel.op.UngroupedAggregateOp; +import io.dingodb.expr.runtime.expr.Expr; +import io.dingodb.expr.runtime.expr.Exprs; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + + +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class RelOpSelectionVisitor extends RelOpVisitorBase { + public static final RelOpSelectionVisitor INSTANCE = new RelOpSelectionVisitor(); + + private static final ExprSelectionVisitor EXPR_SELECTION = ExprSelectionVisitor.INSTANCE; + + @SneakyThrows + private static @Nullable SelectionFlag visitAggList(@NonNull List aggList, @NonNull SelectionObj selected) { + for (Expr expr : aggList) { + if (EXPR_SELECTION.visit(expr, selected.getSelection()) != SelectionFlag.OK) { + return null; + } + } + return SelectionFlag.OK; + } + + @Override + public SelectionFlag visitSourceOp(SourceOp op, @NonNull SelectionObj selected) { + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitFilterOp(@NonNull FilterOp op, @NonNull SelectionObj selected) { + if (EXPR_SELECTION.visit(op.getFilter(), selected.getSelection()) == SelectionFlag.OK) { + selected.setProject(false); + return SelectionFlag.OK; + } + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitProjectOp(@NonNull ProjectOp op, @NonNull SelectionObj selected) { + for (Expr expr : op.getProjects()) { + if (EXPR_SELECTION.visit(expr, selected.getSelection()) != SelectionFlag.OK) { + return null; + } + } + selected.setProject(true); + return SelectionFlag.OK; + } + + @Override + public SelectionFlag visitTandemOp(@NonNull TandemOp op, @NonNull SelectionObj selected) { + if (visit(op.getInput(), selected) == SelectionFlag.OK) { + return visit(op.getOutput(), selected); + } + return null; + } + + @SneakyThrows + @Override + public SelectionFlag visitUngroupedAggregateOp(@NonNull UngroupedAggregateOp op, @NonNull SelectionObj selected) { + return visitAggList(op.getAggList(), selected); + } + + @SneakyThrows + @Override + public SelectionFlag visitGroupedAggregateOp(@NonNull GroupedAggregateOp op, @NonNull SelectionObj selected) { + EXPR_SELECTION.visit(Exprs.val(op.getGroupIndices()), selected.getSelection()); + return visitAggList(op.getAggList(), selected); + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionFlag.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionFlag.java new file mode 100644 index 0000000000..b03e2a72ac --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionFlag.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +public final class SelectionFlag { + public static final SelectionFlag OK = new SelectionFlag(); + + private SelectionFlag() { + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionMapping.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionMapping.java new file mode 100644 index 0000000000..b68432432c --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionMapping.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import java.util.List; +import java.util.OptionalInt; +import java.util.stream.IntStream; + +public class SelectionMapping { + + public static int findIndex(List selection, Integer index) { + OptionalInt newIndex = IntStream.range(0, selection.size()) + .filter(i -> selection.get(i).equals(index)) + .findFirst(); + if (newIndex.isPresent()) { + return newIndex.getAsInt(); + } else { + throw new IllegalArgumentException("Can't find index " + index + " in selection " + selection); + } + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionObj.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionObj.java new file mode 100644 index 0000000000..f63a980954 --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/SelectionObj.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import java.util.Set; + +@Getter +@AllArgsConstructor +public class SelectionObj { + Set selection; + @Setter + boolean isProject; +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ValMappingVisitor.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ValMappingVisitor.java new file mode 100644 index 0000000000..a3ce03bf2e --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ValMappingVisitor.java @@ -0,0 +1,97 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import io.dingodb.expr.common.type.ArrayType; +import io.dingodb.expr.common.type.BoolType; +import io.dingodb.expr.common.type.DateType; +import io.dingodb.expr.common.type.DoubleType; +import io.dingodb.expr.common.type.FloatType; +import io.dingodb.expr.common.type.IntType; +import io.dingodb.expr.common.type.LongType; +import io.dingodb.expr.common.type.StringType; +import io.dingodb.expr.common.type.TypeVisitorBase; +import io.dingodb.expr.runtime.expr.Expr; +import io.dingodb.expr.runtime.expr.Exprs; +import io.dingodb.expr.runtime.expr.Val; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.checkerframework.checker.nullness.qual.NonNull; + +import java.sql.Date; +import java.util.List; + +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +class ValMappingVisitor extends TypeVisitorBase> { + + private final Val val; + + @SneakyThrows + @Override + public Expr visitIntType(@NonNull IntType type, List selection) { + Integer value = (Integer) val.getValue(); + return Exprs.val(value, type); + } + + @SneakyThrows + @Override + public Expr visitLongType(@NonNull LongType type, List selection) { + Long value = (Long) val.getValue(); + return Exprs.val(value, type); + } + + @SneakyThrows + @Override + public Expr visitDateType(@NonNull DateType type, List selection) { + Date value = (Date) val.getValue(); + return Exprs.val(value, type); + } + + @SneakyThrows + @Override + public Expr visitFloatType(@NonNull FloatType type, List selection) { + Float value = (Float) val.getValue(); + return Exprs.val(value, type); + } + + @SneakyThrows + @Override + public Expr visitDoubleType(@NonNull DoubleType type, List selection) { + Double value = (Double) val.getValue(); + return Exprs.val(value, type); + } + + @SneakyThrows + @Override + public Expr visitBoolType(@NonNull BoolType type, List selection) { + Boolean value = (Boolean) val.getValue(); + return Exprs.val(value, type); + } + + @SneakyThrows + @Override + public Expr visitStringType(@NonNull StringType type, List selection) { + String value = (String) val.getValue(); + return Exprs.val(value, type); + } + + @Override + public Expr visitArrayType(@NonNull ArrayType type, List selection) { + return Exprs.val(val.getValue(), type); + } +} diff --git a/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ValSelectionVisitor.java b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ValSelectionVisitor.java new file mode 100644 index 0000000000..b5d3022a50 --- /dev/null +++ b/dingo-exec/src/main/java/io/dingodb/exec/utils/relop/ValSelectionVisitor.java @@ -0,0 +1,95 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.exec.utils.relop; + +import io.dingodb.expr.common.type.ArrayType; +import io.dingodb.expr.common.type.BoolType; +import io.dingodb.expr.common.type.DateType; +import io.dingodb.expr.common.type.DoubleType; +import io.dingodb.expr.common.type.FloatType; +import io.dingodb.expr.common.type.IntType; +import io.dingodb.expr.common.type.LongType; +import io.dingodb.expr.common.type.StringType; +import io.dingodb.expr.common.type.TypeVisitorBase; +import io.dingodb.expr.runtime.expr.Val; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.checkerframework.checker.nullness.qual.NonNull; + +import java.util.Set; + +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +class ValSelectionVisitor extends TypeVisitorBase> { + + private final Val val; + + @SneakyThrows + @Override + public SelectionFlag visitIntType(@NonNull IntType type, Set selected) { + Integer value = (Integer) val.getValue(); + if (value != null) { + selected.add(value); + } + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitLongType(@NonNull LongType type, Set selected) { + Long value = (Long) val.getValue(); + if (value != null) { + selected.add(Math.toIntExact(value)); + } + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitDateType(@NonNull DateType type, Set selected) { + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitFloatType(@NonNull FloatType type, Set selected) { + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitDoubleType(@NonNull DoubleType type, Set selected) { + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitBoolType(@NonNull BoolType type, Set selected) { + return SelectionFlag.OK; + } + + @SneakyThrows + @Override + public SelectionFlag visitStringType(@NonNull StringType type, Set selected) { + return SelectionFlag.OK; + } + + @Override + public SelectionFlag visitArrayType(@NonNull ArrayType type, Set selected) { + return SelectionFlag.OK; + } +}