From cd9c2d22292d52f9164e0d8ca6eb5d2c2604cc1f Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Wed, 19 Jun 2024 23:51:54 +0800 Subject: [PATCH] fix --- .../org/apache/doris/catalog/Database.java | 34 ++++++++++++------- .../org/apache/doris/catalog/JdbcTable.java | 11 ------ .../org/apache/doris/catalog/OdbcTable.java | 11 ------ .../apache/doris/catalog/PartitionInfo.java | 2 ++ .../java/org/apache/doris/catalog/Table.java | 19 ++++++++++- .../java/org/apache/doris/catalog/View.java | 10 ++++-- 6 files changed, 50 insertions(+), 37 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index b074d65cc17d95d..f06f8262aa228c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -35,6 +35,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Preconditions; @@ -76,7 +77,7 @@ * if the table has never been loaded * if the table loading failed on the * previous attempt */ -public class Database extends MetaObject implements Writable, DatabaseIf { +public class Database extends MetaObject implements Writable, DatabaseIf
, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Database.class); private static final String TRANSACTION_QUOTA_SIZE = "transactionQuotaSize"; @@ -595,17 +596,26 @@ public static Database read(DataInput in) throws IOException { db.readFields(in); return db; } else { - Database db = GsonUtils.GSON.fromJson(Text.readString(in), Database.class); - db.fullQualifiedName = ClusterNamespace.getNameFromFullName(db.fullQualifiedName); - db.nameToTable.forEach((tn, tb) -> { - tb.setQualifiedDbName(db.fullQualifiedName); - if (tb instanceof MTMV) { - Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) tb, db.id); - } - db.idToTable.put(tb.getId(), tb); - db.lowerCaseToTableName.put(tn.toLowerCase(), tn); - }); - return db; + return GsonUtils.GSON.fromJson(Text.readString(in), Database.class); + } + } + + @Override + public void gsonPostProcess() throws IOException { + fullQualifiedName = ClusterNamespace.getNameFromFullName(fullQualifiedName); + nameToTable.forEach((tn, tb) -> { + tb.setQualifiedDbName(fullQualifiedName); + if (tb instanceof MTMV) { + Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) tb, id); + } + idToTable.put(tb.getId(), tb); + lowerCaseToTableName.put(tn.toLowerCase(), tn); + }); + + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_105) { + transactionQuotaSize = Config.default_db_max_running_txn_num == -1L + ? Config.max_running_txn_num_per_db + : Config.default_db_max_running_txn_num; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index 8976ba652ed041f..34fe4ba095ad898 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -20,10 +20,8 @@ import org.apache.doris.catalog.Resource.ResourceType; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; -import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.DeepCopy; import org.apache.doris.common.io.Text; -import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TJdbcTable; import org.apache.doris.thrift.TOdbcTableType; import org.apache.doris.thrift.TTableDescriptor; @@ -355,15 +353,6 @@ public JdbcTable clone() { return copied; } - public static JdbcTable read(DataInput in) throws IOException { - if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_136) { - JdbcTable t = new JdbcTable(); - t.readFields(in); - return t; - } - return (JdbcTable) GsonUtils.GSON.fromJson(Text.readString(in), Table.class); - } - private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of jdbc table, " diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java index 2c471adda401a09..eb54ddf73aebefc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java @@ -19,11 +19,9 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; -import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.DeepCopy; import org.apache.doris.common.io.Text; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TOdbcTable; import org.apache.doris.thrift.TOdbcTableType; @@ -387,15 +385,6 @@ public OdbcTable clone() { return copied; } - public static OdbcTable read(DataInput in) throws IOException { - if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_136) { - OdbcTable t = new OdbcTable(); - t.readFields(in); - return t; - } - return (OdbcTable) GsonUtils.GSON.fromJson(Text.readString(in), Table.class); - } - public void resetIdsForRestore(Env env) { id = env.getNextId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index f9edfdb28e191dc..10c01f5f2dea353 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -65,7 +65,9 @@ public class PartitionInfo implements Writable { @SerializedName("pc") protected List partitionColumns = Lists.newArrayList(); // formal partition id -> partition item + @SerializedName("IdToItem") protected Map idToItem = Maps.newHashMap(); + @SerializedName("IdToTempItem") // temp partition id -> partition item protected Map idToTempItem = Maps.newHashMap(); // partition id -> data property diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 06515d462b6e46d..77b6632fa6af216 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -30,6 +30,7 @@ import org.apache.doris.common.util.QueryableReentrantReadWriteLock; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; @@ -64,7 +65,7 @@ /** * Internal representation of table-related metadata. A table contains several partitions. */ -public abstract class Table extends MetaObject implements Writable, TableIf { +public abstract class Table extends MetaObject implements Writable, TableIf, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Table.class); // empirical value. @@ -474,6 +475,22 @@ public static Table read(DataInput in) throws IOException { } } + @Override + public void gsonPostProcess() throws IOException { + List keys = Lists.newArrayList(); + + for (Column column : fullSchema) { + if (column.isKey()) { + keys.add(column); + } + this.nameToColumn.put(column.getName(), column); + } + if (keys.size() > 1) { + keys.forEach(key -> key.setCompoundKey(true)); + hasCompoundKey = true; + } + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java index 391e9a6cfb0f0f2..3d9a6e1cfe49605 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java @@ -28,6 +28,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Preconditions; @@ -51,7 +52,7 @@ * Refreshing or invalidating a view will reload the view's definition but will not * affect the metadata of the underlying tables (if any). */ -public class View extends Table { +public class View extends Table implements GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(View.class); // The original SQL-string given as view definition. Set during analysis. @@ -247,13 +248,18 @@ public static View read(DataInput in) throws IOException { t.readFields(in); return t; } - return (View) GsonUtils.GSON.fromJson(Text.readString(in), Table.class); + return GsonUtils.GSON.fromJson(Text.readString(in), View.class); } public void resetIdsForRestore(Env env) { id = env.getNextId(); } + @Override + public void gsonPostProcess() throws IOException { + originalViewDef = ""; + } + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in);