diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java index 0c58ddd66e5f3f..2a5fb0549b3f9b 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java @@ -35,6 +35,8 @@ public class DefaultValueExprDef implements GsonPostProcessable { @SerializedName("precision") private Long precision; + @SerializedName("exprSql") + private String exprSql; public DefaultValueExprDef(String exprName) { this.exprName = exprName; @@ -45,7 +47,20 @@ public DefaultValueExprDef(String exprName, Long precision) { this.precision = precision; } + public static DefaultValueExprDef fromSql(String exprSql) { + DefaultValueExprDef def = new DefaultValueExprDef((String) null); + def.exprSql = exprSql; + return def; + } + + public boolean isExpressionSql() { + return exprSql != null; + } + public String getSql() { + if (exprSql != null) { + return exprSql; + } StringBuilder sb = new StringBuilder(); sb.append(exprName); sb.append("("); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 597a006707ed2c..0eb71e3a02f85b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -29,10 +29,30 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -314,6 +334,13 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV Preconditions.checkArgument(type.isScalarType()); ScalarType scalarType = (ScalarType) type; + if (defaultValueExprDef != null && defaultValueExprDef.isExpressionSql()) { + validateDefaultValueExpressionSql(type, defaultValue); + String foldedLiteralDefaultValue = computeRealDefaultValueForExpressionSql(type, defaultValue); + validateDefaultValue(type, foldedLiteralDefaultValue, null); + return; + } + // check if default value is valid. // first, check if the type of defaultValue matches primitiveType. // if not check it first, some literal constructor will throw AnalysisException, @@ -450,6 +477,80 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV } } + private static void validateDefaultValueExpressionSql(Type type, String defaultExprSql) + throws AnalysisException { + Expression parsedExpr = new NereidsParser().parseExpression(defaultExprSql); + + if (parsedExpr.anyMatch(e -> e instanceof UnboundSlot)) { + throw new AnalysisException("Default value expression cannot contain column reference: " + defaultExprSql); + } + if (parsedExpr.anyMatch(e -> e instanceof SubqueryExpr)) { + throw new AnalysisException("Default value expression cannot contain subquery: " + defaultExprSql); + } + + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()), null, true, true); + Expression expr; + try { + expr = analyzer.analyze(parsedExpr); + } catch (org.apache.doris.nereids.exceptions.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + + if (expr.anyMatch(e -> e instanceof Slot)) { + throw new AnalysisException("Default value expression cannot contain column reference: " + defaultExprSql); + } + if (expr.anyMatch(e -> e instanceof SubqueryExpr)) { + throw new AnalysisException("Default value expression cannot contain subquery: " + defaultExprSql); + } + if (ExpressionUtils.hasNonWindowAggregateFunction(expr)) { + throw new AnalysisException( + "Default value expression cannot contain aggregate function: " + defaultExprSql); + } + if (expr.anyMatch(e -> e instanceof WindowExpression)) { + throw new AnalysisException("Default value expression cannot contain window function: " + defaultExprSql); + } + if (expr.anyMatch(e -> e instanceof Udf)) { + throw new AnalysisException("Default value expression cannot contain UDF: " + defaultExprSql); + } + + java.util.Set allowedNonDeterministic = ImmutableSet.of( + "now", "current_timestamp", "localtime", "localtimestamp", "current_date"); + if (expr.anyMatch(e -> e instanceof BoundFunction + && !((ExpressionTrait) e).isDeterministic() + && !allowedNonDeterministic.contains(((BoundFunction) e).getName().toLowerCase()))) { + throw new AnalysisException("Default value expression contains non-deterministic function other than " + + "now/current_timestamp/current_date: " + defaultExprSql); + } + + DataType targetType = DataType.fromCatalogType(type); + TypeCoercionUtils.castIfNotSameType(expr, targetType); + } + + private static String computeRealDefaultValueForExpressionSql(Type type, String defaultExprSql) + throws AnalysisException { + Expression parsedExpr = new NereidsParser().parseExpression(defaultExprSql); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()), null, true, true); + Expression expr; + try { + expr = analyzer.analyze(parsedExpr); + } catch (org.apache.doris.nereids.exceptions.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + + DataType targetType = DataType.fromCatalogType(type); + expr = TypeCoercionUtils.castIfNotSameType(expr, targetType); + + ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(CascadesContext.initTempContext()); + Expression folded = FoldConstantRuleOnFE.evaluate(expr, rewriteContext); + + if (!(folded instanceof Literal) || folded instanceof NullLiteral) { + throw new AnalysisException("Default value expression must be foldable to a non-null literal: " + + defaultExprSql); + } + + return ((Literal) folded).getStringValue(); + } + public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("`").append(name).append("` "); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 970999ef80f50e..0c817d660442bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.MaterializedViewHandler; import org.apache.doris.analysis.ColumnDef; import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.backup.Status; import org.apache.doris.backup.Status.ErrCode; @@ -3225,6 +3226,13 @@ public void validateForFlexiblePartialUpdate() throws UserException { } } + public boolean hasExpressionDefaultValue() { + return getFullSchema().stream().anyMatch(col -> { + DefaultValueExprDef exprDef = col.getDefaultValueExprDef(); + return exprDef != null && exprDef.isExpressionSql(); + }); + } + public boolean getEnableUniqueKeyMergeOnWrite() { if (tableProperty == null) { return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 3223ff913594e1..e78dfe0c613e49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2077,6 +2077,7 @@ protected void modifyCommonJobProperties(Map jobProperties) thro this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); + validateExpressionDefaultValueForPartialUpdateForAlter(uniqueKeyUpdateMode); } if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { @@ -2089,6 +2090,7 @@ protected void modifyCommonJobProperties(Map jobProperties) thro } this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); + validateExpressionDefaultValueForPartialUpdateForAlter(uniqueKeyUpdateMode); } } @@ -2140,4 +2142,31 @@ private void validateFlexiblePartialUpdateForAlter() throws UserException { throw new DdlException("Flexible partial update does not support COLUMNS specification"); } } + + private void validateExpressionDefaultValueForPartialUpdateForAlter(TUniqueKeyUpdateMode mode) + throws UserException { + ConnectContext ctx = ConnectContext.get(); + if (ctx == null) { + return; + } + if (mode == TUniqueKeyUpdateMode.UPSERT + || ctx.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) { + return; + } + Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null) { + throw new DdlException("Database not found: " + dbId); + } + Table table = db.getTableNullable(tableId); + if (table == null) { + throw new DdlException("Table not found: " + tableId); + } + if (!(table instanceof OlapTable)) { + return; + } + if (((OlapTable) table).hasExpressionDefaultValue()) { + throw new DdlException("Can't do partial update on merge-on-write Unique table with " + + "expression default value column while `allow_partial_update_with_expression_default` is false."); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java index 507241b2a6c76c..fb6c77cd94350c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java @@ -73,6 +73,7 @@ import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.planner.GroupCommitBlockSink; import org.apache.doris.planner.OlapTableSink; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileScanRangeParams; @@ -275,6 +276,15 @@ public NereidsLoadPlanInfoCollector(OlapTable destTable, NereidsLoadTaskInfo tas */ public LoadPlanInfo collectLoadPlanInfo(LogicalPlan logicalPlan, DescriptorTable descTable, TupleDescriptor scanDescriptor) { + ConnectContext ctx = ConnectContext.get(); + if (uniquekeyUpdateMode != TUniqueKeyUpdateMode.UPSERT + && destTable.hasExpressionDefaultValue() + && ctx != null + && !ctx.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) { + throw new AnalysisException("Can't do partial update on merge-on-write Unique table with " + + "expression default value column while `allow_partial_update_with_expression_default` is false."); + } + this.logicalPlan = logicalPlan; CascadesContext cascadesContext = CascadesContext.initContext(new StatementContext(), logicalPlan, PhysicalProperties.ANY); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index b01b62db5f33e5..4272d4b3ed93c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4190,6 +4190,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { defaultValue = Optional.of(DefaultValue.E_NUM_DEFAULT_VALUE); } else if (ctx.BITMAP_EMPTY() != null) { defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); + } else if (ctx.defaultExpr != null) { + defaultValue = Optional.of(DefaultValue.expressionSqlDefaultValue(getOriginSql(ctx.defaultExpr))); } } if (ctx.UPDATE() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java index 4e717bbb1d6382..c3c22635091506 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java @@ -47,6 +47,7 @@ import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; @@ -188,6 +189,15 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer && partialUpdateColNameToExpression.size() <= targetTable.getFullSchema().size() * 3 / 10 && !targetTable.isUniqKeyMergeOnWriteWithClusterKeys(); + if (isPartialUpdate + && targetTable.hasExpressionDefaultValue() + && !ctx.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) { + throw new AnalysisException("Partial update is not supported for table with expression default value. " + + "You can set session variable '" + + SessionVariable.ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT + + "'=true to bypass this check (may be unsafe). "); + } + List partialUpdateColNames = new ArrayList<>(); List partialUpdateSelectItems = new ArrayList<>(); if (isPartialUpdate) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 39e26090be65b3..0a1ad159c40170 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -26,7 +26,16 @@ import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.util.SqlUtils; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; @@ -39,6 +48,7 @@ import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.SessionVariable; @@ -525,17 +535,25 @@ public void validate(boolean isOlap, Set keysSet, Set clusterKey * translate to catalog create table stmt */ public Column translateToCatalogStyle() { + String defaultValueStr = defaultValue.map(DefaultValue::getValue).orElse(null); + DefaultValueExprDef defaultExprDef = defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null); + String realDefaultValueStr = defaultValueStr; + + if (defaultValue.isPresent() && defaultExprDef != null && defaultExprDef.isExpressionSql()) { + realDefaultValueStr = computeRealDefaultValueForExpressionSql(type, defaultValueStr); + } + Column column = new Column(name, type.toCatalogDataType(), isKey, aggType, isNullable, - autoIncInitValue, defaultValue.map(DefaultValue::getValue).orElse(null), comment, isVisible, - defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), Column.COLUMN_UNIQUE_ID_INIT_VALUE, - defaultValue.map(DefaultValue::getValue).orElse(null), onUpdateDefaultValue.isPresent(), + autoIncInitValue, defaultValueStr, comment, isVisible, + defaultExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, + realDefaultValueStr, onUpdateDefaultValue.isPresent(), onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId, generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null), generatedColumnsThatReferToThis, generatedColumnDesc.map(desc -> - ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) - .orElse(null) - ); + ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) + .orElse(null) + ); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } @@ -544,20 +562,46 @@ public Column translateToCatalogStyle() { * translate to catalog column for schema change */ public Column translateToCatalogStyleForSchemaChange() { + String defaultValueStr = defaultValue.map(DefaultValue::getValue).orElse(null); + DefaultValueExprDef defaultExprDef = defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null); + String realDefaultValueStr = defaultValue.map(DefaultValue::getRawValue).orElse(null); + + if (defaultValue.isPresent() && defaultExprDef != null && defaultExprDef.isExpressionSql()) { + realDefaultValueStr = computeRealDefaultValueForExpressionSql(type, defaultValueStr); + } + Column column = new Column(name, type.toCatalogDataType(), isKey, aggType, isNullable, - autoIncInitValue, defaultValue.map(DefaultValue::getValue).orElse(null), comment, isVisible, - defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), Column.COLUMN_UNIQUE_ID_INIT_VALUE, - defaultValue.map(DefaultValue::getRawValue).orElse(null), onUpdateDefaultValue.isPresent(), + autoIncInitValue, defaultValueStr, comment, isVisible, + defaultExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, + realDefaultValueStr, onUpdateDefaultValue.isPresent(), onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId, generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null), generatedColumnsThatReferToThis, generatedColumnDesc.map(desc -> - ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) - .orElse(null)); + ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) + .orElse(null) + ); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } + private static String computeRealDefaultValueForExpressionSql(DataType targetType, String defaultExprSql) { + Expression expr = new NereidsParser().parseExpression(defaultExprSql); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()), null, true, true); + expr = analyzer.analyze(expr); + + expr = TypeCoercionUtils.castIfNotSameType(expr, targetType); + + ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(CascadesContext.initTempContext()); + Expression folded = FoldConstantRuleOnFE.evaluate(expr, rewriteContext); + + if (!(folded instanceof Literal) || folded instanceof NullLiteral) { + throw new AnalysisException("Default value expression must be foldable to a non-null literal: " + + defaultExprSql); + } + return ((Literal) folded).getStringValue(); + } + /** * add hidden column */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java index e8a75c0299b4d6..9d25139d5dd6ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java @@ -440,6 +440,17 @@ private void checkDBTable(ConnectContext ctx) throws AnalysisException { if (isPartialUpdate && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) { throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW"); } + + if ((isPartialUpdate || uniqueKeyUpdateMode != TUniqueKeyUpdateMode.UPSERT) + && table instanceof OlapTable + && ((OlapTable) table).hasExpressionDefaultValue() + && !ctx.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) { + throw new AnalysisException("Partial update is not supported for table with expression default value. " + + "You can set session variable '" + + org.apache.doris.qe.SessionVariable.ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT + + "'=true to bypass this check (may be unsafe). "); + } + // Validate flexible partial update constraints if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { validateFlexiblePartialUpdate((OlapTable) table); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java index ad5f41ec2d1816..87090340388a2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java @@ -77,6 +77,15 @@ public DefaultValue(String value, String exprName, Long precision) { this.defaultValueExprDef = new DefaultValueExprDef(exprName, precision); } + private DefaultValue(String value, DefaultValueExprDef defaultValueExprDef) { + this.value = value; + this.defaultValueExprDef = defaultValueExprDef; + } + + public static DefaultValue expressionSqlDefaultValue(String exprSql) { + return new DefaultValue(exprSql, DefaultValueExprDef.fromSql(exprSql)); + } + /** * default value current_timestamp(precision) */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index fa5e34046d1c80..85248a80a2f9c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -362,6 +362,21 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, } } } + + UnboundTableSink tableSink = (UnboundTableSink) unboundLogicalSink; + if (tableSink.isPartialUpdate()) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null + && olapTable.hasExpressionDefaultValue() + && !connectContext.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) { + throw new AnalysisException( + "Partial update is not supported for table with expression default value. " + + "You can set session variable '" + + SessionVariable.ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT + + "'=true to bypass this check " + + "(then missing columns are filled using the pre-folded literal value). "); + } + } } } Plan query = unboundLogicalSink.child(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 5eacd86e5b0d6f..d749d06129b9d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -619,6 +619,9 @@ public String toString() { public static final String ENABLE_UNIQUE_KEY_PARTIAL_UPDATE = "enable_unique_key_partial_update"; + public static final String ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT = + "allow_partial_update_with_expression_default"; + public static final String PARTIAL_UPDATE_NEW_KEY_BEHAVIOR = "partial_update_new_key_behavior"; public static final String INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD = "inverted_index_conjunction_opt_threshold"; @@ -2659,6 +2662,14 @@ public static boolean isEagerAggregationOnJoin() { @VarAttrDef.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = true) public boolean enableUniqueKeyPartialUpdate = false; + @VarAttrDef.VarAttr(name = ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT, needForward = true, + description = {"当表包含 DEFAULT <表达式> 列默认值时，是否允许部分列更新/导入。" + + "开启可能导致部分场景使用 DDL 时刻折叠的静态默认值。", + "Whether to allow partial update/load when table contains DEFAULT column defaults." + + " Enabling it may cause some paths to use the static literal folded at DDL time."}) + public boolean allowPartialUpdateWithExpressionDefault = false; + + @VarAttrDef.VarAttr(name = PARTIAL_UPDATE_NEW_KEY_BEHAVIOR, needForward = true, description = { "用于设置部分列更新中对于新插入的行的行为", "Used to set the behavior for newly inserted rows in partial update." @@ -5374,6 +5385,14 @@ public void setEnableUniqueKeyPartialUpdate(boolean enableUniqueKeyPartialUpdate this.enableUniqueKeyPartialUpdate = enableUniqueKeyPartialUpdate; } + public boolean isAllowPartialUpdateWithExpressionDefault() { + return allowPartialUpdateWithExpressionDefault; + } + + public void setAllowPartialUpdateWithExpressionDefault(boolean allowPartialUpdateWithExpressionDefault) { + this.allowPartialUpdateWithExpressionDefault = allowPartialUpdateWithExpressionDefault; + } + public TPartialUpdateNewRowPolicy getPartialUpdateNewRowPolicy() { return parsePartialUpdateNewKeyBehavior(partialUpdateNewKeyPolicy); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DefaultValueExpressionValidatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DefaultValueExpressionValidatorTest.java new file mode 100644 index 00000000000000..216587583e67af --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DefaultValueExpressionValidatorTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DefaultValueExpressionValidatorTest extends TestWithFeService { + + @Override + public void runBeforeAll() throws Exception { + createDatabase("test"); + connectContext.setDatabase("test"); + } + + @Test + public void testValidateDefaultValueExpressionSqlOk() throws AnalysisException { + ColumnDef.validateDefaultValue(ScalarType.DATEV2, + "to_date(now())", + DefaultValueExprDef.fromSql("to_date(now())")); + + ColumnDef.validateDefaultValue(ScalarType.createDatetimeV2Type(3), + "now(3)", + DefaultValueExprDef.fromSql("now(3)")); + } + + @Test + public void testValidateDefaultValueExpressionSqlRejectsColumnReference() { + AnalysisException ex = Assertions.assertThrows(AnalysisException.class, + () -> ColumnDef.validateDefaultValue(Type.INT, + "k1 + 1", + DefaultValueExprDef.fromSql("k1 + 1"))); + Assertions.assertTrue(ex.getMessage().contains("column reference")); + } + + @Test + public void testValidateDefaultValueExpressionSqlRejectsRand() { + AnalysisException ex = Assertions.assertThrows(AnalysisException.class, + () -> ColumnDef.validateDefaultValue(Type.DOUBLE, + "rand()", + DefaultValueExprDef.fromSql("rand()"))); + Assertions.assertTrue(ex.getMessage().toLowerCase().contains("non-deterministic")); + } + + @Test + public void testValidateDefaultValueExpressionSqlRejectsAggregateFunction() { + AnalysisException ex = Assertions.assertThrows(AnalysisException.class, + () -> ColumnDef.validateDefaultValue(Type.BIGINT, + "sum(1)", + DefaultValueExprDef.fromSql("sum(1)"))); + String msg = ex.getMessage().toLowerCase(); + Assertions.assertTrue(msg.contains("aggregate") || msg.contains("sum")); + } + + @Test + public void testValidateDefaultValueExpressionSqlRejectsTooLongBoundedCharAndVarchar() { + AnalysisException varcharEx = Assertions.assertThrows(AnalysisException.class, + () -> ColumnDef.validateDefaultValue(ScalarType.createVarcharType(3), + "concat('abcdef')", + DefaultValueExprDef.fromSql("concat('abcdef')"))); + Assertions.assertTrue(varcharEx.getMessage().toLowerCase().contains("too long")); + + AnalysisException charEx = Assertions.assertThrows(AnalysisException.class, + () -> ColumnDef.validateDefaultValue(ScalarType.createCharType(3), + "concat('abcdef')", + DefaultValueExprDef.fromSql("concat('abcdef')"))); + Assertions.assertTrue(charEx.getMessage().toLowerCase().contains("too long")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ExpressionDefaultValueOriginSqlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ExpressionDefaultValueOriginSqlTest.java new file mode 100644 index 00000000000000..ad0cdf7f230617 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ExpressionDefaultValueOriginSqlTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ExpressionDefaultValueOriginSqlTest extends TestWithFeService { + + @Override + public void runBeforeAll() throws Exception { + createDatabase("test_expr_default_origin_sql"); + connectContext.setDatabase("test_expr_default_origin_sql"); + } + + @Test + public void testCreateTableWithCastExpressionDefaultPreservesWhitespace() { + String tableSql = "create table test_expr_default_origin_sql.tbl_default_cast_expr (\n" + + " k1 int not null,\n" + + " v1 int not null default CAST('1' AS INT)\n" + + ")\n" + + "unique key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties(\"replication_num\"=\"1\")"; + + Assertions.assertDoesNotThrow(() -> createTable(tableSql)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/ExpressionDefaultPartialUpdateGuardTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/ExpressionDefaultPartialUpdateGuardTest.java new file mode 100644 index 00000000000000..5f3553f0d65335 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/ExpressionDefaultPartialUpdateGuardTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TUniqueId; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +public class ExpressionDefaultPartialUpdateGuardTest extends TestWithFeService { + + @Override + public void runBeforeAll() throws Exception { + createDatabase("test"); + connectContext.setDatabase("test"); + + String tableSql = "create table test.tbl_expr_default_mow (\n" + + " k1 int not null,\n" + + " v1 int null,\n" + + " d datev2 not null default to_date(now())\n" + + ")\n" + + "unique key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties(\"replication_num\"=\"1\",\"enable_unique_key_merge_on_write\"=\"true\")"; + createTable(tableSql); + } + + private static boolean anyCauseMessageContains(Throwable t, String substring) { + while (t != null) { + String message = t.getMessage(); + if (message != null && message.contains(substring)) { + return true; + } + t = t.getCause(); + } + return false; + } + + private void analyzeInsertWithoutTxn(String sql) throws Exception { + connectContext.setThreadLocalInfo(); + + StatementContext statementContext = createStatementCtx(sql); + LogicalPlan parsedPlan = new NereidsParser().parseSingle(sql); + + UUID uuid = UUID.randomUUID(); + connectContext.setQueryId(new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits())); + + InsertIntoTableCommand insertIntoTableCommand = (InsertIntoTableCommand) parsedPlan; + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(parsedPlan, statementContext); + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, logicalPlanAdapter); + + insertIntoTableCommand.initPlan(connectContext, stmtExecutor, false); + } + + @Test + public void testInsertPartialUpdateRejectedWhenExpressionDefaultAndSessionVarDisabled() { + connectContext.getSessionVariable().setEnableUniqueKeyPartialUpdate(true); + connectContext.getSessionVariable().setAllowPartialUpdateWithExpressionDefault(false); + + Throwable t = Assertions.assertThrows(Throwable.class, + () -> analyzeInsertWithoutTxn("insert into tbl_expr_default_mow(k1) values (1)")); + + Assertions.assertTrue( + anyCauseMessageContains(t, "Partial update is not supported for table with expression default value"), + () -> "Unexpected exception: " + t + ); + } + + @Test + public void testInsertPartialUpdateAllowedWhenSessionVarEnabled() { + connectContext.getSessionVariable().setEnableUniqueKeyPartialUpdate(true); + connectContext.getSessionVariable().setAllowPartialUpdateWithExpressionDefault(true); + + Assertions.assertDoesNotThrow( + () -> analyzeInsertWithoutTxn("insert into tbl_expr_default_mow(k1) values (1)")); + } + + @Test + public void testInsertValuesDowngradedToUpsertNotRejectedWhenExpressionDefaultAndSessionVarDisabled() { + connectContext.getSessionVariable().setEnableUniqueKeyPartialUpdate(true); + connectContext.getSessionVariable().setAllowPartialUpdateWithExpressionDefault(false); + + Assertions.assertDoesNotThrow( + () -> analyzeInsertWithoutTxn( + "insert into tbl_expr_default_mow values (1, 2, to_date('2024-01-01'))")); + } +} diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index cd2391b1775f1e..39894d04522ff5 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1549,7 +1549,8 @@ columnDef ((NOT)? nullable=NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? (DEFAULT (nullValue=NULL | SUBTRACT? INTEGER_VALUE | SUBTRACT? DECIMAL_VALUE | PI | E | BITMAP_EMPTY | stringValue=STRING_LITERAL - | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? + | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)? + | defaultExpr=expression))? (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? ; diff --git a/regression-test/data/nereids_syntax_p0/default_value_expression.out b/regression-test/data/nereids_syntax_p0/default_value_expression.out new file mode 100644 index 00000000000000..1cf8adba1b6faf --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/default_value_expression.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !default_expr_select -- +1 true true true + +-- !default_expr_partial_update_bypass -- +1 true true diff --git a/regression-test/suites/nereids_syntax_p0/default_value_expression.groovy b/regression-test/suites/nereids_syntax_p0/default_value_expression.groovy new file mode 100644 index 00000000000000..3bac2d1fa428b1 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/default_value_expression.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nereids_default_value_expression", "p0") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql "DROP TABLE IF EXISTS tbl_default_expr" + sql """ + CREATE TABLE tbl_default_expr ( + id INT NOT NULL, + d DATEV2 NOT NULL DEFAULT to_date(now()), + dt DATETIMEV2(3) NOT NULL DEFAULT now(3), + s STRING NOT NULL DEFAULT concat('a-', DATE_FORMAT(now(), '%M %e, %Y')) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + sql "INSERT INTO tbl_default_expr(id) VALUES (1)" + sql "sync" + qt_default_expr_select "SELECT id, d = CURRENT_DATE, dt IS NOT NULL, s = concat('a-', DATE_FORMAT(now(), '%M %e, %Y')) FROM tbl_default_expr ORDER BY id" + + test { + sql "DROP TABLE IF EXISTS tbl_default_expr_bad_ref" + sql """ + CREATE TABLE tbl_default_expr_bad_ref ( + k1 INT, + v1 INT DEFAULT k1 + 1 + ) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + exception "Default value expression cannot contain column reference" + } + + test { + sql "DROP TABLE IF EXISTS tbl_default_expr_bad_rand" + sql """ + CREATE TABLE tbl_default_expr_bad_rand ( + k1 INT, + v1 DOUBLE DEFAULT rand() + ) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + exception "non-deterministic" + } + + try { + sql "DROP TABLE IF EXISTS tbl_default_expr_mow" + sql """ + CREATE TABLE tbl_default_expr_mow ( + k1 INT, + v1 INT NULL, + d DATEV2 NOT NULL DEFAULT to_date(now()) + ) + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql "set enable_unique_key_partial_update=true" + sql "set allow_partial_update_with_expression_default=false" + + test { + sql "INSERT INTO tbl_default_expr_mow(k1) VALUES (1)" + exception "Partial update is not supported for table with expression default value" + } + + sql "set allow_partial_update_with_expression_default=true" + sql "INSERT INTO tbl_default_expr_mow(k1) VALUES (1)" + sql "sync" + qt_default_expr_partial_update_bypass "SELECT k1, v1 IS NULL, d IS NOT NULL FROM tbl_default_expr_mow ORDER BY k1" + } finally { + sql "set allow_partial_update_with_expression_default=false" + sql "set enable_unique_key_partial_update=false" + sql "sync" + } +}