diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 9948e7fd8cf472..67653242535e96 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -174,6 +174,7 @@ Status ColumnReader::create_struct(const ColumnReaderOptions& opts, const Column std::unique_ptr struct_reader( new ColumnReader(opts, meta, num_rows, file_reader)); struct_reader->_sub_readers.reserve(meta.children_columns_size()); + // now we support struct column can add the children columns according to the schema-change behavior for (size_t i = 0; i < meta.children_columns_size(); i++) { std::unique_ptr sub_reader; RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(i), @@ -714,7 +715,7 @@ Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterat return Status::OK(); } -Status ColumnReader::new_iterator(ColumnIterator** iterator) { +Status ColumnReader::new_iterator(ColumnIterator** iterator, const TabletColumn* tablet_column) { if (is_empty()) { *iterator = new EmptyFileColumnIterator(); return Status::OK(); @@ -729,13 +730,13 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) { return new_agg_state_iterator(iterator); } case FieldType::OLAP_FIELD_TYPE_STRUCT: { - return new_struct_iterator(iterator); + return new_struct_iterator(iterator, tablet_column); } case FieldType::OLAP_FIELD_TYPE_ARRAY: { - return new_array_iterator(iterator); + return new_array_iterator(iterator, tablet_column); } case FieldType::OLAP_FIELD_TYPE_MAP: { - return new_map_iterator(iterator); + return new_map_iterator(iterator, tablet_column); } case FieldType::OLAP_FIELD_TYPE_VARIANT: { *iterator = new VariantRootColumnIterator(new FileColumnIterator(this)); @@ -753,55 +754,71 @@ Status ColumnReader::new_agg_state_iterator(ColumnIterator** iterator) { return Status::OK(); } -Status ColumnReader::new_array_iterator(ColumnIterator** iterator) { +Status ColumnReader::new_array_iterator(ColumnIterator** iterator, + const TabletColumn* tablet_column) { ColumnIterator* item_iterator = nullptr; - RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&item_iterator)); + RETURN_IF_ERROR(_sub_readers[0]->new_iterator( + &item_iterator, tablet_column ? &tablet_column->get_sub_column(0) : nullptr)); ColumnIterator* offset_iterator = nullptr; - RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&offset_iterator)); + RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&offset_iterator, tablet_column)); auto* ofcIter = new OffsetFileColumnIterator(reinterpret_cast(offset_iterator)); ColumnIterator* null_iterator = nullptr; if (is_nullable()) { - RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator)); + RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator, tablet_column)); } *iterator = new ArrayFileColumnIterator(this, ofcIter, item_iterator, null_iterator); return Status::OK(); } -Status ColumnReader::new_map_iterator(ColumnIterator** iterator) { +Status ColumnReader::new_map_iterator(ColumnIterator** iterator, + const TabletColumn* tablet_column) { ColumnIterator* key_iterator = nullptr; - RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator)); + RETURN_IF_ERROR(_sub_readers[0]->new_iterator( + &key_iterator, tablet_column ? &tablet_column->get_sub_column(0) : nullptr)); ColumnIterator* val_iterator = nullptr; - RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator)); + RETURN_IF_ERROR(_sub_readers[1]->new_iterator( + &val_iterator, tablet_column ? &tablet_column->get_sub_column(1) : nullptr)); ColumnIterator* offsets_iterator = nullptr; - RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator)); + RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator, nullptr)); auto* ofcIter = new OffsetFileColumnIterator(reinterpret_cast(offsets_iterator)); ColumnIterator* null_iterator = nullptr; if (is_nullable()) { - RETURN_IF_ERROR(_sub_readers[3]->new_iterator(&null_iterator)); + RETURN_IF_ERROR(_sub_readers[3]->new_iterator(&null_iterator, nullptr)); } *iterator = new MapFileColumnIterator(this, null_iterator, ofcIter, key_iterator, val_iterator); return Status::OK(); } -Status ColumnReader::new_struct_iterator(ColumnIterator** iterator) { +Status ColumnReader::new_struct_iterator(ColumnIterator** iterator, + const TabletColumn* tablet_column) { std::vector sub_column_iterators; size_t child_size = is_nullable() ? _sub_readers.size() - 1 : _sub_readers.size(); + size_t tablet_column_size = tablet_column ? tablet_column->get_sub_columns().size() : 0; sub_column_iterators.reserve(child_size); ColumnIterator* sub_column_iterator; for (size_t i = 0; i < child_size; i++) { - RETURN_IF_ERROR(_sub_readers[i]->new_iterator(&sub_column_iterator)); + RETURN_IF_ERROR(_sub_readers[i]->new_iterator( + &sub_column_iterator, tablet_column ? &tablet_column->get_sub_column(i) : nullptr)); sub_column_iterators.push_back(sub_column_iterator); } + // create default_iterator for schema-change behavior which increase column + for (size_t i = child_size; i < tablet_column_size; i++) { + TabletColumn column = tablet_column->get_sub_column(i); + std::unique_ptr* it = new std::unique_ptr(); + RETURN_IF_ERROR(Segment::new_default_iterator(column, it)); + sub_column_iterators.push_back(it->get()); + } + ColumnIterator* null_iterator = nullptr; if (is_nullable()) { - RETURN_IF_ERROR(_sub_readers[child_size]->new_iterator(&null_iterator)); + RETURN_IF_ERROR(_sub_readers[child_size]->new_iterator(&null_iterator, nullptr)); } *iterator = new StructFileColumnIterator(this, null_iterator, sub_column_iterators); return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 2afc269a86c012..c2f3d3822d2efc 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -134,10 +134,10 @@ class ColumnReader : public MetadataAdder { virtual ~ColumnReader(); // create a new column iterator. Client should delete returned iterator - Status new_iterator(ColumnIterator** iterator); - Status new_array_iterator(ColumnIterator** iterator); - Status new_struct_iterator(ColumnIterator** iterator); - Status new_map_iterator(ColumnIterator** iterator); + Status new_iterator(ColumnIterator** iterator, const TabletColumn* tablet_column); + Status new_array_iterator(ColumnIterator** iterator, const TabletColumn* tablet_column); + Status new_struct_iterator(ColumnIterator** iterator, const TabletColumn* tablet_column); + Status new_map_iterator(ColumnIterator** iterator, const TabletColumn* tablet_column); Status new_agg_state_iterator(ColumnIterator** iterator); // Client should delete returned iterator Status new_bitmap_index_iterator(BitmapIndexIterator** iterator); diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp index fe7167e9444a76..9635ffc1ec1199 100644 --- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp +++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp @@ -56,7 +56,7 @@ Status HierarchicalDataReader::create(std::unique_ptr* reader, // like {"a" : "b" : {"e" : 1.1}} in jsonb format if (read_type == ReadType::MERGE_SPARSE) { ColumnIterator* it; - RETURN_IF_ERROR(root->data.reader->new_iterator(&it)); + RETURN_IF_ERROR(root->data.reader->new_iterator(&it, nullptr)); stream_iter->set_root(std::make_unique( root->data.file_column_type->create_column(), std::unique_ptr(it), root->data.file_column_type)); @@ -132,7 +132,7 @@ Status HierarchicalDataReader::add_stream(const SubcolumnColumnReaders::Node* no } CHECK(node); ColumnIterator* it; - RETURN_IF_ERROR(node->data.reader->new_iterator(&it)); + RETURN_IF_ERROR(node->data.reader->new_iterator(&it, nullptr)); std::unique_ptr it_ptr; it_ptr.reset(it); SubstreamIterator reader(node->data.file_column_type->create_column(), std::move(it_ptr), diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index b5ab3f0e873549..0235082ade825b 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -702,32 +702,12 @@ Status Segment::_create_column_readers(const SegmentFooterPB& footer) { return Status::OK(); } -static Status new_default_iterator(const TabletColumn& tablet_column, - std::unique_ptr* iter) { - if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) { - return Status::InternalError( - "invalid nonexistent column without default value. column_uid={}, column_name={}, " - "column_type={}", - tablet_column.unique_id(), tablet_column.name(), tablet_column.type()); - } - auto type_info = get_type_info(&tablet_column); - std::unique_ptr default_value_iter(new DefaultValueColumnIterator( - tablet_column.has_default_value(), tablet_column.default_value(), - tablet_column.is_nullable(), std::move(type_info), tablet_column.precision(), - tablet_column.frac())); - ColumnIteratorOptions iter_opts; - - RETURN_IF_ERROR(default_value_iter->init(iter_opts)); - *iter = std::move(default_value_iter); - return Status::OK(); -} - Status Segment::_new_iterator_with_variant_root(const TabletColumn& tablet_column, std::unique_ptr* iter, const SubcolumnColumnReaders::Node* root, vectorized::DataTypePtr target_type_hint) { ColumnIterator* it; - RETURN_IF_ERROR(root->data.reader->new_iterator(&it)); + RETURN_IF_ERROR(root->data.reader->new_iterator(&it, &tablet_column)); auto* stream_iter = new ExtractReader( tablet_column, std::make_unique(root->data.file_column_type->create_column(), @@ -794,7 +774,7 @@ Status Segment::new_column_iterator_with_path(const TabletColumn& tablet_column, assert(leaf); std::unique_ptr sibling_iter; ColumnIterator* sibling_iter_ptr; - RETURN_IF_ERROR(leaf->data.reader->new_iterator(&sibling_iter_ptr)); + RETURN_IF_ERROR(leaf->data.reader->new_iterator(&sibling_iter_ptr, &tablet_column)); sibling_iter.reset(sibling_iter_ptr); *iter = std::make_unique(std::move(sibling_iter), leaf->data.file_column_type); @@ -825,7 +805,7 @@ Status Segment::new_column_iterator_with_path(const TabletColumn& tablet_column, return Status::OK(); } ColumnIterator* it; - RETURN_IF_ERROR(node->data.reader->new_iterator(&it)); + RETURN_IF_ERROR(node->data.reader->new_iterator(&it, &tablet_column)); iter->reset(it); return Status::OK(); } @@ -836,7 +816,7 @@ Status Segment::new_column_iterator_with_path(const TabletColumn& tablet_column, // Direct read extracted columns const auto* node = _sub_column_tree[unique_id].find_leaf(relative_path); ColumnIterator* it; - RETURN_IF_ERROR(node->data.reader->new_iterator(&it)); + RETURN_IF_ERROR(node->data.reader->new_iterator(&it, &tablet_column)); iter->reset(it); } else { // Node contains column with children columns or has correspoding sparse columns @@ -890,7 +870,8 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column, } // init iterator by unique id ColumnIterator* it; - RETURN_IF_ERROR(_column_readers.at(tablet_column.unique_id())->new_iterator(&it)); + RETURN_IF_ERROR( + _column_readers.at(tablet_column.unique_id())->new_iterator(&it, &tablet_column)); iter->reset(it); if (config::enable_column_type_check && !tablet_column.is_agg_state_type() && @@ -909,7 +890,8 @@ Status Segment::new_column_iterator(int32_t unique_id, const StorageReadOptions* std::unique_ptr* iter) { RETURN_IF_ERROR(_create_column_readers_once(opt->stats)); ColumnIterator* it; - RETURN_IF_ERROR(_column_readers.at(unique_id)->new_iterator(&it)); + TabletColumn tablet_column = _tablet_schema->column_by_uid(unique_id); + RETURN_IF_ERROR(_column_readers.at(unique_id)->new_iterator(&it, &tablet_column)); iter->reset(it); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index 441ae3e85e9b3f..c626b125424846 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -79,6 +79,28 @@ using SegmentSharedPtr = std::shared_ptr; // change finished, client should disable all cached Segment for old TabletSchema. class Segment : public std::enable_shared_from_this, public MetadataAdder { public: + static Status new_default_iterator(const TabletColumn& tablet_column, + std::unique_ptr* iter) { + if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) { + return Status::InternalError( + "invalid nonexistent column without default value. column_uid={}, " + "column_name={}, " + "column_type={}", + tablet_column.unique_id(), tablet_column.name(), tablet_column.type()); + } + auto type_info = get_type_info(&tablet_column); + std::unique_ptr default_value_iter( + new DefaultValueColumnIterator(tablet_column.has_default_value(), + tablet_column.default_value(), + tablet_column.is_nullable(), std::move(type_info), + tablet_column.precision(), tablet_column.frac())); + ColumnIteratorOptions iter_opts; + + RETURN_IF_ERROR(default_value_iter->init(iter_opts)); + *iter = std::move(default_value_iter); + return Status::OK(); + } + static Status open(io::FileSystemSPtr fs, const std::string& path, int64_t tablet_id, uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema, const io::FileReaderOptions& reader_options, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java index 234c0d4eef0efb..3b6f6fd49dc8cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -196,10 +196,14 @@ public static void checkSupportSchemaChangeForComplexType(Type checkType, Type o if (checkType.isStructType() && other.isStructType()) { StructType thisStructType = (StructType) checkType; StructType otherStructType = (StructType) other; - if (thisStructType.getFields().size() != otherStructType.getFields().size()) { - throw new DdlException("Cannot change struct type with different field size"); + if (thisStructType.getFields().size() > otherStructType.getFields().size()) { + throw new DdlException("Cannot change " + checkType.toSql() + " to " + other.toSql()); } for (int i = 0; i < thisStructType.getFields().size(); i++) { + // do not support struct same position field name change + if (!thisStructType.getFields().get(i).getName().equals(otherStructType.getFields().get(i).getName())) { + throw new DdlException("Cannot change " + checkType.toSql() + " to " + other.toSql()); + } checkSupportSchemaChangeForComplexType(thisStructType.getFields().get(i).getType(), otherStructType.getFields().get(i).getType(), true); } diff --git a/regression-test/data/schema_change_p0/test_modify_struct.out b/regression-test/data/schema_change_p0/test_modify_struct.out new file mode 100644 index 00000000000000..fb136cba8bf766 --- /dev/null +++ b/regression-test/data/schema_change_p0/test_modify_struct.out @@ -0,0 +1,52 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sc_before -- +0 {"col":"commiter"} +1 {"col":"amory"} + +-- !sc_after -- +0 {"col":"commiter"} +1 {"col":"amory"} +2 {"col":"amoryIsBetter"} + +-- !sc_after1 -- +0 {"col":"commiter", "col1":null} +1 {"col":"amory", "col1":null} +2 {"col":"amoryIsBetter", "col1":null} +3 {"col":"amoryIsBetterCommiter", "col1":1} + +-- !sc_before -- +1 1 {"col":"xxx"} +2 2 {"col":"yyy"} +3 3 {"col":"zzz"} + +-- !sc_after -- +1 1 {"col":"xxx", "col1":null} +2 2 {"col":"yyy", "col1":null} +3 3 {"col":"zzz", "col1":null} +3 4 {"col":"amoryIsBetterCommiter", "col1":1} +4 4 {"col":"amoryIsBetterCommiterAgain", "col1":2} + +-- !sc_before -- +1 1 {"col":"xxx"} +2 2 {"col":"yyy"} +3 3 {"col":"zzz"} + +-- !sc_after -- +1 1 {"col":"xxx", "col1":null} +2 2 {"col":"yyy", "col1":null} +3 3 {"col":"zzz", "col1":null} +3 4 {"col":"amoryIsBetterCommiter", "col1":1} +4 4 {"col":"amoryIsBetterCommiterAgain", "col1":2} + +-- !sc_before -- +1 1 {"col":"xxx"} +2 2 {"col":"yyy"} +3 3 {"col":"zzz"} + +-- !sc_after -- +1 1 {"col":"xxx", "col1":null} +2 2 {"col":"yyy", "col1":null} +3 3 {"col":"zzz", "col1":null} +3 4 {"col":"amoryIsBetterCommiter", "col1":1} +4 4 {"col":"amoryIsBetterCommiterAgain", "col1":2} + diff --git a/regression-test/suites/schema_change_p0/test_modify_struct.groovy b/regression-test/suites/schema_change_p0/test_modify_struct.groovy new file mode 100644 index 00000000000000..71a2d730f8b1a5 --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_modify_struct.groovy @@ -0,0 +1,406 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite ("test_modify_struct") { + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + return jobStateResult[0][9] + } + + def tableNamePrefix = "test_struct_add_sub_column" + def tableName = tableNamePrefix + int max_try_secs = 300 + + try { + + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + // duplicate table + tableName = tableNamePrefix + "_dup" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `c0` LARGEINT NOT NULL, + `c_s` STRUCT + ) DISTRIBUTED BY HASH(c0) BUCKETS 1 + PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" ) + """ + + sql """ insert into $tableName values + (0, named_struct('col','commiter')); + """ + sql """ insert into $tableName values + (1, named_struct('col','amory')); + """ + // this can be insert but with cut off the left string to 10 + test { + sql """ insert into $tableName values + (11, named_struct('col','amoryIsBetter')); + """ + exception "Insert has filtered data in strict mode" + } + + + String[][] res = sql """ desc ${tableName} """ + logger.info(res[1][1]) + assertEquals(res[1][1].toLowerCase(),"struct") + + order_qt_sc_before " select * from ${tableName} order by c0; " + + // modify struct with varchar(20) + sql """ alter table ${tableName} modify column c_s STRUCT """ + Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { + String result = getJobState(tableName) + if (result == "FINISHED") { + return true; + } + return false; + }); + // check the struct column + res = sql """ desc ${tableName} """ + logger.info(res[1][1]) + assertEquals(res[1][1].toLowerCase(),"struct") + + // insert some data to modified struct with varchar(20) + sql """ insert into ${tableName} values + (2, named_struct('col','amoryIsBetter')); + """ + + order_qt_sc_after " select * from ${tableName} order by c0; " + + + // test struct add sub column + sql """ alter table ${tableName} modify column c_s STRUCT """ + Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { + String result = getJobState(tableName) + if (result == "FINISHED") { + return true; + } + return false; + }); + // check the struct column + res = sql """ desc ${tableName} """ + logger.info(res[1][1]) + assertEquals(res[1][1].toLowerCase(),"struct") + + // insert some data to modified struct with varchar(30) and int + sql """ insert into ${tableName} values + (3, named_struct('col','amoryIsBetterCommiter', 'col1', 1)); + """ + + order_qt_sc_after1 " select * from ${tableName} order by c0; " + + // test struct reduce sub-column behavior not support + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test struct sub-column type change + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test struct sub-column name change + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test unique key table + tableName = tableNamePrefix + "_unique" + sql "DROP TABLE IF EXISTS ${tableName} FORCE;" + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` + ( + `siteid` INT DEFAULT '10', + `citycode` SMALLINT, + `c_s` STRUCT, + ) + UNIQUE KEY(`siteid`, `citycode`) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, 1, named_struct('col','xxx')), + (2, 2, named_struct('col','yyy')), + (3, 3, named_struct('col','zzz')); + """ + + // check struct column + res = sql """ desc ${tableName} """ + logger.info(res[2][1]) + assertEquals(res[2][1].toLowerCase(),"struct") + + // test unique key table modify struct column + order_qt_sc_before " select * from ${tableName} order by siteid, citycode; " + + + // test struct add sub column + sql """ alter table ${tableName} modify column c_s STRUCT """ + Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { + String result = getJobState(tableName) + if (result == "FINISHED") { + return true; + } + return false; + }); + // check the struct column + res = sql """ desc ${tableName} """ + logger.info(res[2][1]) + assertEquals(res[2][1].toLowerCase(),"struct") + + // insert some data to modified struct with varchar(30) and int + sql """ insert into ${tableName} values + (3, 4, named_struct('col','amoryIsBetterCommiter', 'col1', 1)); + """ + sql """ insert into ${tableName} values + (4, 4, named_struct('col','amoryIsBetterCommiterAgain', 'col1', 2)); + """ + + + order_qt_sc_after " select * from ${tableName} order by siteid, citycode; " + + // test struct reduce sub-column behavior not support + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test struct sub-column type change + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test struct sub-column name change + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test MOW table + tableName = tableNamePrefix + "_mow" + sql "DROP TABLE IF EXISTS ${tableName} FORCE;" + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` + ( + `siteid` INT DEFAULT '10', + `citycode` SMALLINT, + `c_s` STRUCT, + ) + UNIQUE KEY(`siteid`, `citycode`) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, 1, named_struct('col','xxx')), + (2, 2, named_struct('col','yyy')), + (3, 3, named_struct('col','zzz')); + """ + + // check struct column + res = sql """ desc ${tableName} """ + logger.info(res[2][1]) + assertEquals(res[2][1].toLowerCase(),"struct") + + // test MOW table modify struct column + order_qt_sc_before " select * from ${tableName} order by siteid, citycode; " + + + // test struct add sub column + sql """ alter table ${tableName} modify column c_s STRUCT """ + Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { + String result = getJobState(tableName) + if (result == "FINISHED") { + return true; + } + return false; + }); + + // check the struct column + res = sql """ desc ${tableName} """ + logger.info(res[2][1]) + assertEquals(res[2][1].toLowerCase(),"struct") + + // insert some data to modified struct with varchar(30) and int + sql """ insert into ${tableName} values + (3, 4, named_struct('col','amoryIsBetterCommiter', 'col1', 1)); + """ + sql """ insert into ${tableName} values + (4, 4, named_struct('col','amoryIsBetterCommiterAgain', 'col1', 2)); + """ + + order_qt_sc_after " select * from ${tableName} order by siteid, citycode; " + + // test struct reduce sub-column behavior not support + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test struct sub-column type change + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test struct sub-column name change + test { + sql """ alter table ${tableName} modify column c_s STRUCT """ + exception "Cannot change" + } + + // test agg table + tableName = tableNamePrefix + "_agg" + sql "DROP TABLE IF EXISTS ${tableName} FORCE;" + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` + ( + `siteid` INT DEFAULT '10', + `citycode` SMALLINT, + `c_s` STRUCT REPLACE, + ) + AGGREGATE KEY(`siteid`, `citycode`) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ INSERT INTO ${tableName} VALUES + (1, 1, named_struct('col','xxx')), + (2, 2, named_struct('col','yyy')), + (3, 3, named_struct('col','zzz')); + """ + + // check struct column + res = sql """ desc ${tableName} """ + logger.info(res[2][1]) + assertEquals(res[2][1].toLowerCase(),"struct") + + // test MOW table modify struct column + order_qt_sc_before " select * from ${tableName} order by siteid, citycode; " + + // test struct add sub column + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE """ + Awaitility.await().atMost(max_try_secs, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { + String result = getJobState(tableName) + if (result == "FINISHED") { + return true; + } + return false; + }); + + // check the struct column + res = sql """ desc ${tableName} """ + logger.info(res[2][1]) + assertEquals(res[2][1].toLowerCase(),"struct") + + // insert some data to modified struct with varchar(30) and int + sql """ insert into ${tableName} values + (3, 4, named_struct('col','amoryIsBetterCommiter', 'col1', 1)); + """ + sql """ insert into ${tableName} values + (4, 4, named_struct('col','amoryIsBetterCommiterAgain', 'col1', 2)); + """ + + order_qt_sc_after " select * from ${tableName} order by siteid, citycode; " + + // test agg table for not change agg type + test { + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE_IF_NOT_NULL """ + exception "Can not change aggregation type" + } + + // test struct reduce sub-column behavior not support + test { + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE""" + exception "Cannot change" + } + + // test struct sub-column type change + test { + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE""" + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE""" + exception "Cannot change" + } + + test { + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE""" + exception "Cannot change" + } + + // test struct sub-column name change + test { + sql """ alter table ${tableName} modify column c_s STRUCT REPLACE""" + exception "Cannot change" + } + + } finally { + try_sql("DROP TABLE IF EXISTS ${tableName}") + } + +}