From 557693c8a9f38314bc96467e3745a83a63416b2d Mon Sep 17 00:00:00 2001 From: amory Date: Fri, 10 Nov 2023 22:18:26 +0800 Subject: [PATCH] [FIX](complextype) fxi array nested struct literal #26270 (#26778) --- .../olap/rowset/segment_v2/column_reader.cpp | 1 + .../org/apache/doris/catalog/ArrayType.java | 10 ++++ .../org/apache/doris/catalog/MapType.java | 15 +++++ .../org/apache/doris/catalog/StructType.java | 24 ++++++++ .../java/org/apache/doris/catalog/Type.java | 27 ++++----- .../cast_function/test_cast_map_function.out | 48 --------------- .../test_cast_map_function.groovy | 60 ------------------- 7 files changed, 60 insertions(+), 125 deletions(-) delete mode 100644 regression-test/data/nereids_function_p0/cast_function/test_cast_map_function.out delete mode 100644 regression-test/suites/nereids_function_p0/cast_function/test_cast_map_function.groovy diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index d9a074e29046cc..3d1e89de2ed49e 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -87,6 +87,7 @@ Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& case FieldType::OLAP_FIELD_TYPE_STRUCT: { // not support empty struct DCHECK(meta.children_columns_size() >= 1); + num_rows = meta.children_columns(0).num_rows(); // create struct column reader std::unique_ptr struct_reader( new ColumnReader(opts, meta, num_rows, file_reader)); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java index 477d278bf8a24b..b820cda7a449b0 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -163,6 +163,16 @@ public static boolean canCastTo(ArrayType type, ArrayType targetType) { return Type.canCastTo(type.getItemType(), targetType.getItemType()); } + public static Type getAssignmentCompatibleType(ArrayType t1, ArrayType t2, boolean strict) { + Type itemCompatibleType = Type.getAssignmentCompatibleType(t1.getItemType(), t2.getItemType(), strict); + + if (itemCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + + return new ArrayType(itemCompatibleType, t1.getContainsNull() || t2.getContainsNull()); + } + @Override public void toThrift(TTypeDesc container) { TTypeNode node = new TTypeNode(); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 691c4d7e04db68..06dba8c2fb0219 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -198,6 +198,21 @@ public static boolean canCastTo(MapType type, MapType targetType) { || targetType.getValueType().isStringType() && type.getValueType().isStringType()); } + public static Type getAssignmentCompatibleType(MapType t1, MapType t2, boolean strict) { + Type keyCompatibleType = Type.getAssignmentCompatibleType(t1.getKeyType(), t2.getKeyType(), strict); + if (keyCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + Type valCompatibleType = Type.getAssignmentCompatibleType(t1.getValueType(), t2.getValueType(), strict); + if (valCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + + return new MapType(keyCompatibleType, valCompatibleType, + t1.getIsKeyContainsNull() || t2.getIsKeyContainsNull(), + t1.getIsValueContainsNull() || t2.getIsValueContainsNull()); + } + @Override public boolean supportSubType(Type subType) { for (Type supportedType : Type.getMapSubTypes()) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java index 0a058ae5406f85..1d6be19d28efbc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java @@ -29,6 +29,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Arrays; @@ -110,6 +111,29 @@ public static boolean canCastTo(StructType type, StructType targetType) { return true; } + public static Type getAssignmentCompatibleType(StructType t1, StructType t2, boolean strict) { + ArrayList fieldsLeft = t1.getFields(); + ArrayList fieldsRight = t2.getFields(); + ArrayList fieldsRes = new ArrayList<>(); + + for (int i = 0; i < t1.getFields().size(); ++i) { + StructField leftField = fieldsLeft.get(i); + StructField rightField = fieldsRight.get(i); + Type itemCompatibleType = Type.getAssignmentCompatibleType(leftField.getType(), rightField.getType(), + strict); + if (itemCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + fieldsRes.add(new StructField(StringUtils.isEmpty(leftField.getName()) ? rightField.getName() + : leftField.getName(), + itemCompatibleType, StringUtils.isEmpty(leftField.getComment()) ? rightField.getComment() + : leftField.getComment(), leftField.getContainsNull() || rightField.getContainsNull())); + + } + + return new StructType(fieldsRes); + } + @Override public boolean isSupported() { for (StructField f : fields) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index fe2c132b9df0dc..b32b29c18fe7fb 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -746,19 +746,14 @@ public static Type getAssignmentCompatibleType(Type t1, Type t2, boolean strict) } if (t1.isArrayType() && t2.isArrayType()) { - ArrayType arrayType1 = (ArrayType) t1; - ArrayType arrayType2 = (ArrayType) t2; - Type itemCompatibleType = Type.getAssignmentCompatibleType(arrayType1.getItemType(), - arrayType2.getItemType(), strict); - - if (itemCompatibleType.isInvalid()) { - return itemCompatibleType; - } - - return new ArrayType(itemCompatibleType, arrayType1.getContainsNull() || arrayType2.getContainsNull()); - } else if (t1.isArrayType() && t2.isNull()) { + return ArrayType.getAssignmentCompatibleType((ArrayType) t1, (ArrayType) t2, strict); + } else if (t1.isMapType() && t2.isMapType()) { + return MapType.getAssignmentCompatibleType((MapType) t1, (MapType) t2, strict); + } else if (t1.isStructType() && t2.isStructType()) { + return StructType.getAssignmentCompatibleType((StructType) t1, (StructType) t2, strict); + } else if (t1.isComplexType() && t2.isNull()) { return t1; - } else if (t1.isNull() && t2.isArrayType()) { + } else if (t1.isNull() && t2.isComplexType()) { return t2; } @@ -1963,17 +1958,15 @@ public static boolean matchExactType(Type type1, Type type2, boolean ignorePreci } else if (type2.isArrayType()) { // For types array, we also need to check contains null for case like // cast(array as array) - if (!((ArrayType) type2).getContainsNull() == ((ArrayType) type1).getContainsNull()) { + if (((ArrayType) type2).getContainsNull() != ((ArrayType) type1).getContainsNull()) { return false; } return matchExactType(((ArrayType) type2).getItemType(), ((ArrayType) type1).getItemType()); } else if (type2.isMapType()) { - // For types array, we also need to check contains null for case like - // cast(array as array) - if (!((MapType) type2).getIsKeyContainsNull() == ((MapType) type1).getIsKeyContainsNull()) { + if (((MapType) type2).getIsKeyContainsNull() != ((MapType) type1).getIsKeyContainsNull()) { return false; } - if (!((MapType) type2).getIsValueContainsNull() == ((MapType) type1).getIsValueContainsNull()) { + if (((MapType) type2).getIsValueContainsNull() != ((MapType) type1).getIsValueContainsNull()) { return false; } return matchExactType(((MapType) type2).getKeyType(), ((MapType) type1).getKeyType()) diff --git a/regression-test/data/nereids_function_p0/cast_function/test_cast_map_function.out b/regression-test/data/nereids_function_p0/cast_function/test_cast_map_function.out deleted file mode 100644 index c9da5a1c286ce9..00000000000000 --- a/regression-test/data/nereids_function_p0/cast_function/test_cast_map_function.out +++ /dev/null @@ -1,48 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select -- -1 {"aa":1, "b":2, "1234567":77} -2 {"b":12, "123":7777} - --- !select -- -{} - --- !select -- -{} - --- !sql1 -- -\N - --- !sql2 -- -{"":NULL} - --- !sql3 -- -{"1":2} - --- !sql4 -- -{"aa":1, "b":2, "1234567":77} -{"b":12, "123":7777} - --- !sql5 -- -{"aa":1, "b":2, "1234567":77} -{"b":12, "123":7777} - --- !sql6 -- -{"aa":1, "b":2, "1234567":77} -{"b":12, "123":97} - --- !sql7 -- -{"aa":"1", "b":"2", "1234567":"77"} -{"b":"12", "123":"7777"} - --- !sql8 -- -{NULL:"1", NULL:"2", 1234567:"77"} -{NULL:"12", 123:"7777"} - --- !sql9 -- -{NULL:1, NULL:2, 1234567:77} -{NULL:12, 123:7777} - --- !sql10 -- -{NULL:NULL, NULL:NULL, 1234567:NULL} -{NULL:NULL, 123:NULL} - diff --git a/regression-test/suites/nereids_function_p0/cast_function/test_cast_map_function.groovy b/regression-test/suites/nereids_function_p0/cast_function/test_cast_map_function.groovy deleted file mode 100644 index e3a4ccdaecc22f..00000000000000 --- a/regression-test/suites/nereids_function_p0/cast_function/test_cast_map_function.groovy +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_cast_map_function", "query") { - sql """ set enable_nereids_planner = true; """ - sql """ set enable_fallback_to_original_planner=false; """ - def tableName = "tbl_test_cast_map_function_nereids" - - sql """DROP TABLE IF EXISTS ${tableName}""" - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `k1` int(11) NULL COMMENT "", - `k2` Map NOT NULL COMMENT "", - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - DISTRIBUTED BY HASH(`k1`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "storage_format" = "V2" - ) - """ - // insert into with implicit cast - sql """ INSERT INTO ${tableName} VALUES(1, {"aa": 1, "b": 2, "1234567": 77}) """ - sql """ INSERT INTO ${tableName} VALUES(2, {"b":12, "123":7777}) """ - - qt_select """ select * from ${tableName} order by k1; """ - - qt_select " select cast({} as MAP);" - qt_select " select cast(map() as MAP); " - qt_sql1 "select cast(NULL as MAP)" - - // literal NONSTRICT_SUPERTYPE_OF cast - qt_sql2 "select cast({'':''} as MAP);" - qt_sql3 "select cast({1:2} as MAP);" - - // select SUPERTYPE_OF cast - qt_sql4 "select cast(k2 as map) from ${tableName} order by k1;" - - // select NONSTRICT_SUPERTYPE_OF cast , this behavior is same with nested scala type - qt_sql5 "select cast(k2 as map) from ${tableName} order by k1;" - qt_sql6 "select cast(k2 as map) from ${tableName} order by k1;" - qt_sql7 "select cast(k2 as map) from ${tableName} order by k1;" - qt_sql8 "select cast(k2 as map) from ${tableName} order by k1;" - qt_sql9 "select cast(k2 as map) from ${tableName} order by k1;" - qt_sql10 "select cast(k2 as map) from ${tableName} order by k1;" -}