[feat](expression support for default value) Add expression-based column default values

fe/fe-catalog/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java

@@ -35,6 +35,8 @@ public class DefaultValueExprDef implements GsonPostProcessable {
     @SerializedName("precision")
     private Long precision;
 
+    @SerializedName("exprSql")
+    private String exprSql;
 
     public DefaultValueExprDef(String exprName) {
         this.exprName = exprName;
@@ -45,7 +47,20 @@ public DefaultValueExprDef(String exprName, Long precision) {
         this.precision = precision;
     }
 
+    public static DefaultValueExprDef fromSql(String exprSql) {
+        DefaultValueExprDef def = new DefaultValueExprDef((String) null);
+        def.exprSql = exprSql;
+        return def;
+    }
+
+    public boolean isExpressionSql() {
+        return exprSql != null;
+    }
+
     public String getSql() {
+        if (exprSql != null) {
+            return exprSql;
+        }
         StringBuilder sb = new StringBuilder();
         sb.append(exprName);
         sb.append("(");

fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java

@@ -29,10 +29,25 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.util.SqlUtils;
 import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.nereids.analyzer.Scope;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
+import org.apache.doris.nereids.trees.expressions.WindowExpression;
+import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.trees.expressions.functions.Udf;
 import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.TypeCoercionUtils;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -314,6 +329,11 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV
         Preconditions.checkArgument(type.isScalarType());
         ScalarType scalarType = (ScalarType) type;
 
+        if (defaultValueExprDef != null && defaultValueExprDef.isExpressionSql()) {
+            validateDefaultValueExpressionSql(type, defaultValue);
+            return;
+        }
+
         // check if default value is valid.
         // first, check if the type of defaultValue matches primitiveType.
         // if not check it first, some literal constructor will throw AnalysisException,
@@ -450,6 +470,55 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV
         }
     }
 
+    private static void validateDefaultValueExpressionSql(Type type, String defaultExprSql)
+            throws AnalysisException {
+        Expression parsedExpr = new NereidsParser().parseExpression(defaultExprSql);
+
+        if (parsedExpr.anyMatch(e -> e instanceof UnboundSlot)) {
+            throw new AnalysisException("Default value expression cannot contain column reference: " + defaultExprSql);
+        }
+        if (parsedExpr.anyMatch(e -> e instanceof SubqueryExpr)) {
+            throw new AnalysisException("Default value expression cannot contain subquery: " + defaultExprSql);
+        }
+
+        ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()), null, true, true);
+        Expression expr;
+        try {
+            expr = analyzer.analyze(parsedExpr);
+        } catch (org.apache.doris.nereids.exceptions.AnalysisException e) {
+            throw new AnalysisException(e.getMessage(), e);
+        }
+
+        if (expr.anyMatch(e -> e instanceof Slot)) {
+            throw new AnalysisException("Default value expression cannot contain column reference: " + defaultExprSql);
+        }
+        if (expr.anyMatch(e -> e instanceof SubqueryExpr)) {
+            throw new AnalysisException("Default value expression cannot contain subquery: " + defaultExprSql);
+        }
+        if (ExpressionUtils.hasNonWindowAggregateFunction(expr)) {
+            throw new AnalysisException(
+                "Default value expression cannot contain aggregate function: " + defaultExprSql);
+        }
+        if (expr.anyMatch(e -> e instanceof WindowExpression)) {
+            throw new AnalysisException("Default value expression cannot contain window function: " + defaultExprSql);
+        }
+        if (expr.anyMatch(e -> e instanceof Udf)) {
+            throw new AnalysisException("Default value expression cannot contain UDF: " + defaultExprSql);
+        }
+
+        java.util.Set<String> allowedNonDeterministic = ImmutableSet.of(
+                "now", "current_timestamp", "localtime", "localtimestamp", "current_date");
+        if (expr.anyMatch(e -> e instanceof BoundFunction
+                && !((ExpressionTrait) e).isDeterministic()
+                && !allowedNonDeterministic.contains(((BoundFunction) e).getName().toLowerCase()))) {
+            throw new AnalysisException("Default value expression contains non-deterministic function other than "
+                + "now/current_timestamp/current_date: " + defaultExprSql);
+        }
+
+        DataType targetType = DataType.fromCatalogType(type);
+        TypeCoercionUtils.castIfNotSameType(expr, targetType);
+    }
+
     public String toSql() {
         StringBuilder sb = new StringBuilder();
         sb.append("`").append(name).append("` ");

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -20,6 +20,7 @@
 import org.apache.doris.alter.MaterializedViewHandler;
 import org.apache.doris.analysis.ColumnDef;
 import org.apache.doris.analysis.DataSortInfo;
+import org.apache.doris.analysis.DefaultValueExprDef;
 import org.apache.doris.analysis.InvertedIndexUtil;
 import org.apache.doris.backup.Status;
 import org.apache.doris.backup.Status.ErrCode;
@@ -3225,6 +3226,13 @@ public void validateForFlexiblePartialUpdate() throws UserException {
         }
     }
 
+    public boolean hasExpressionDefaultValue() {
+        return getFullSchema().stream().anyMatch(col -> {
+            DefaultValueExprDef exprDef = col.getDefaultValueExprDef();
+            return exprDef != null && exprDef.isExpressionSql();
+        });
+    }
+
     public boolean getEnableUniqueKeyMergeOnWrite() {
         if (tableProperty == null) {
             return false;

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java

@@ -4189,6 +4189,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) {
                 defaultValue = Optional.of(DefaultValue.E_NUM_DEFAULT_VALUE);
             } else if (ctx.BITMAP_EMPTY() != null) {
                 defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
+            } else if (ctx.defaultExpr != null) {
+                defaultValue = Optional.of(DefaultValue.expressionSqlDefaultValue(ctx.defaultExpr.getText()));
             }
         }
         if (ctx.UPDATE() != null) {

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java

@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.util.RelationUtil;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.qe.StmtExecutor;
 import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
 
@@ -188,6 +189,15 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer
                 && partialUpdateColNameToExpression.size() <= targetTable.getFullSchema().size() * 3 / 10
                 && !targetTable.isUniqKeyMergeOnWriteWithClusterKeys();
 
+        if (isPartialUpdate
+                && targetTable.hasExpressionDefaultValue()
+                && !ctx.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) {
+            throw new AnalysisException("Partial update is not supported for table with expression default value. "
+                    + "You can set session variable '"
+                    + SessionVariable.ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT
+                    + "'=true to bypass this check (may be unsafe). ");
+        }
+
         List<String> partialUpdateColNames = new ArrayList<>();
         List<NamedExpression> partialUpdateSelectItems = new ArrayList<>();
         if (isPartialUpdate) {

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java

@@ -26,7 +26,16 @@
 import org.apache.doris.common.CaseSensibility;
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.common.util.SqlUtils;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.analyzer.Scope;
 import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.types.ArrayType;
 import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.BitmapType;
@@ -39,6 +48,7 @@
 import org.apache.doris.nereids.types.TinyIntType;
 import org.apache.doris.nereids.types.VarcharType;
 import org.apache.doris.nereids.types.coercion.CharacterType;
+import org.apache.doris.nereids.util.TypeCoercionUtils;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.ConnectContextUtil;
 import org.apache.doris.qe.SessionVariable;
@@ -525,17 +535,25 @@ public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKey
      * translate to catalog create table stmt
      */
     public Column translateToCatalogStyle() {
+        String defaultValueStr = defaultValue.map(DefaultValue::getValue).orElse(null);
+        DefaultValueExprDef defaultExprDef = defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null);
+        String realDefaultValueStr = defaultValueStr;
+
+        if (defaultValue.isPresent() && defaultExprDef != null && defaultExprDef.isExpressionSql()) {
+            realDefaultValueStr = computeRealDefaultValueForExpressionSql(type, defaultValueStr);
+        }
+
         Column column = new Column(name, type.toCatalogDataType(), isKey, aggType, isNullable,
-                autoIncInitValue, defaultValue.map(DefaultValue::getValue).orElse(null), comment, isVisible,
-                defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), Column.COLUMN_UNIQUE_ID_INIT_VALUE,
-                defaultValue.map(DefaultValue::getValue).orElse(null), onUpdateDefaultValue.isPresent(),
+                autoIncInitValue, defaultValueStr, comment, isVisible,
+                defaultExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE,
+                realDefaultValueStr, onUpdateDefaultValue.isPresent(),
                 onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId,
                 generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null),
                 generatedColumnsThatReferToThis,
                 generatedColumnDesc.map(desc ->
-                        ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get()))
-                        .orElse(null)
-                );
+                    ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get()))
+                .orElse(null)
+        );
         column.setAggregationTypeImplicit(aggTypeImplicit);
         return column;
     }
@@ -544,20 +562,46 @@ public Column translateToCatalogStyle() {
      * translate to catalog column for schema change
      */
     public Column translateToCatalogStyleForSchemaChange() {
+        String defaultValueStr = defaultValue.map(DefaultValue::getValue).orElse(null);
+        DefaultValueExprDef defaultExprDef = defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null);
+        String realDefaultValueStr = defaultValue.map(DefaultValue::getRawValue).orElse(null);
+
+        if (defaultValue.isPresent() && defaultExprDef != null && defaultExprDef.isExpressionSql()) {
+            realDefaultValueStr = computeRealDefaultValueForExpressionSql(type, defaultValueStr);
+        }
+
         Column column = new Column(name, type.toCatalogDataType(), isKey, aggType, isNullable,
-                autoIncInitValue, defaultValue.map(DefaultValue::getValue).orElse(null), comment, isVisible,
-                defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), Column.COLUMN_UNIQUE_ID_INIT_VALUE,
-                defaultValue.map(DefaultValue::getRawValue).orElse(null), onUpdateDefaultValue.isPresent(),
+                autoIncInitValue, defaultValueStr, comment, isVisible,
+                defaultExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE,
+                realDefaultValueStr, onUpdateDefaultValue.isPresent(),
                 onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId,
                 generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null),
                 generatedColumnsThatReferToThis,
                 generatedColumnDesc.map(desc ->
-                        ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get()))
-                        .orElse(null));
+                    ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get()))
+                .orElse(null)
+        );
         column.setAggregationTypeImplicit(aggTypeImplicit);
         return column;
     }
 
+    private static String computeRealDefaultValueForExpressionSql(DataType targetType, String defaultExprSql) {
+        Expression expr = new NereidsParser().parseExpression(defaultExprSql);
+        ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()), null, true, true);
+        expr = analyzer.analyze(expr);
+
+        expr = TypeCoercionUtils.castIfNotSameType(expr, targetType);
+
+        ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(CascadesContext.initTempContext());
+        Expression folded = FoldConstantRuleOnFE.evaluate(expr, rewriteContext);
+
+        if (!(folded instanceof Literal) || folded instanceof NullLiteral) {
+            throw new AnalysisException("Default value expression must be foldable to a non-null literal: "
+                + defaultExprSql);
+        }
+        return ((Literal) folded).getStringValue();
+    }
+
     /**
      * add hidden column
      */

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java

@@ -440,6 +440,17 @@ private void checkDBTable(ConnectContext ctx) throws AnalysisException {
         if (isPartialUpdate && !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) {
             throw new AnalysisException("load by PARTIAL_COLUMNS is only supported in unique table MoW");
         }
+
+        if ((isPartialUpdate || uniqueKeyUpdateMode != TUniqueKeyUpdateMode.UPSERT)
+                && table instanceof OlapTable
+                && ((OlapTable) table).hasExpressionDefaultValue()
+                && !ctx.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) {
+            throw new AnalysisException("Partial update is not supported for table with expression default value. "
+                    + "You can set session variable '"
+                    + org.apache.doris.qe.SessionVariable.ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT
+                    + "'=true to bypass this check (may be unsafe). ");
+        }
+
         // Validate flexible partial update constraints
         if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) {
             validateFlexiblePartialUpdate((OlapTable) table);

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java

@@ -77,6 +77,15 @@ public DefaultValue(String value, String exprName, Long precision) {
         this.defaultValueExprDef = new DefaultValueExprDef(exprName, precision);
     }
 
+    private DefaultValue(String value, DefaultValueExprDef defaultValueExprDef) {
+        this.value = value;
+        this.defaultValueExprDef = defaultValueExprDef;
+    }
+
+    public static DefaultValue expressionSqlDefaultValue(String exprSql) {
+        return new DefaultValue(exprSql, DefaultValueExprDef.fromSql(exprSql));
+    }
+
     /**
      * default value current_timestamp(precision)
      */

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java

@@ -311,6 +311,17 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table,
                 // check the necessary conditions for partial updates
                 OlapTable olapTable = (OlapTable) table;
 
+                ConnectContext connectContext = ConnectContext.get();
+                if (connectContext != null
+                        && olapTable.hasExpressionDefaultValue()
+                        && !connectContext.getSessionVariable().isAllowPartialUpdateWithExpressionDefault()) {
+                    throw new AnalysisException(
+                        "Partial update is not supported for table with expression default value. "
+                        + "You can set session variable '"
+                        + SessionVariable.ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT
+                        + "'=true to bypass this check (may be unsafe). ");
+                }
+
                 if (!olapTable.getEnableUniqueKeyMergeOnWrite() || olapTable.isUniqKeyMergeOnWriteWithClusterKeys()) {
                     // when enable_unique_key_partial_update = true,
                     // only unique table with MOW (and without cluster keys)

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

@@ -619,6 +619,9 @@ public String toString() {
 
     public static final String ENABLE_UNIQUE_KEY_PARTIAL_UPDATE = "enable_unique_key_partial_update";
 
+    public static final String ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT =
+            "allow_partial_update_with_expression_default";
+
     public static final String PARTIAL_UPDATE_NEW_KEY_BEHAVIOR = "partial_update_new_key_behavior";
 
     public static final String INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD = "inverted_index_conjunction_opt_threshold";
@@ -2655,6 +2658,14 @@ public static boolean isEagerAggregationOnJoin() {
     @VarAttrDef.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = true)
     public boolean enableUniqueKeyPartialUpdate = false;
 
+    @VarAttrDef.VarAttr(name = ALLOW_PARTIAL_UPDATE_WITH_EXPRESSION_DEFAULT, needForward = true,
+            description = {"当表包含 DEFAULT <表达式> 列默认值时，是否允许部分列更新/导入。"
+                    + "开启可能导致部分场景使用 DDL 时刻折叠的静态默认值。",
+                    "Whether to allow partial update/load when table contains DEFAULT <expression> column defaults."
+                    + " Enabling it may cause some paths to use the static literal folded at DDL time."})
+    public boolean allowPartialUpdateWithExpressionDefault = false;
+
+
     @VarAttrDef.VarAttr(name = PARTIAL_UPDATE_NEW_KEY_BEHAVIOR, needForward = true, description = {
             "用于设置部分列更新中对于新插入的行的行为",
             "Used to set the behavior for newly inserted rows in partial update."
@@ -5338,6 +5349,14 @@ public void setEnableUniqueKeyPartialUpdate(boolean enableUniqueKeyPartialUpdate
         this.enableUniqueKeyPartialUpdate = enableUniqueKeyPartialUpdate;
     }
 
+    public boolean isAllowPartialUpdateWithExpressionDefault() {
+        return allowPartialUpdateWithExpressionDefault;
+    }
+
+    public void setAllowPartialUpdateWithExpressionDefault(boolean allowPartialUpdateWithExpressionDefault) {
+        this.allowPartialUpdateWithExpressionDefault = allowPartialUpdateWithExpressionDefault;
+    }
+
     public TPartialUpdateNewRowPolicy getPartialUpdateNewRowPolicy() {
         return parsePartialUpdateNewKeyBehavior(partialUpdateNewKeyPolicy);
     }