diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 7aa98625639636..919917e1ec5919 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -97,8 +97,10 @@ public final class FeMetaVersion { // For mate gson public static final int VERSION_137 = 137; + public static final int VERSION_138 = 138; + // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_137; + public static final int VERSION_CURRENT = VERSION_138; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index a0f5527ef6d73d..e0f94f115311f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -91,7 +91,6 @@ public class Database extends MetaObject implements Writable, DatabaseIf, // table family group map private final Map idToTable; - @SerializedName(value = "nameToTable") private ConcurrentMap nameToTable; // table name lower cast -> table name private final Map lowerCaseToTableName; @@ -595,8 +594,35 @@ public static Database read(DataInput in) throws IOException { Database db = new Database(); db.readFields(in); return db; - } else { + } else if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_138) { return GsonUtils.GSON.fromJson(Text.readString(in), Database.class); + } else { + Database db = GsonUtils.GSON.fromJson(Text.readString(in), Database.class); + db.readTables(in); + return db; + } + } + + private void writeTables(DataOutput out) throws IOException { + out.writeInt(nameToTable.size()); + for (Table table : nameToTable.values()) { + Text.writeString(out, GsonUtils.GSON.toJson(table)); + } + } + + private void readTables(DataInput in) throws IOException { + nameToTable = Maps.newConcurrentMap(); + int numTables = in.readInt(); + for (int i = 0; i < numTables; ++i) { + Table table = Table.read(in); + table.setQualifiedDbName(fullQualifiedName); + if (table instanceof MTMV) { + Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) table, id); + } + String tableName = table.getName(); + nameToTable.put(tableName, table); + idToTable.put(table.getId(), table); + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } @@ -650,6 +676,10 @@ public String getSignature(int signatureVersion) { public void write(DataOutput out) throws IOException { discardHudiTable(); Text.writeString(out, GsonUtils.GSON.toJson(this)); + if (FeMetaVersion.VERSION_138 <= Env.getCurrentEnvJournalVersion()) { + writeTables(out); + return; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d5b1c258c5ddf7..a66539c5ab75ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -135,44 +135,44 @@ public enum OlapTableState { WAITING_STABLE } - @SerializedName(value = "state") + @SerializedName(value = "s", alternate = {"state"}) private volatile OlapTableState state; // index id -> index meta - @SerializedName("indexIdToMeta") + @SerializedName(value = "itm", alternate = {"indexIdToMeta"}) private Map indexIdToMeta = Maps.newHashMap(); // index name -> index id - @SerializedName("indexNameToId") + @SerializedName(value = "inti", alternate = {"indexNameToId"}) private Map indexNameToId = Maps.newHashMap(); - @SerializedName("keysType") + @SerializedName(value = "kt", alternate = {"keysType"}) private KeysType keysType; @Setter - @SerializedName("partitionInfo") + @SerializedName(value = "pi", alternate = {"partitionInfo"}) private PartitionInfo partitionInfo; - @SerializedName("idToPartition") + @SerializedName(value = "itp", alternate = {"idToPartition"}) @Getter private ConcurrentHashMap idToPartition = new ConcurrentHashMap<>(); // handled in postgsonprocess @Getter private Map nameToPartition = Maps.newTreeMap(); - @SerializedName(value = "distributionInfo") + @SerializedName(value = "di", alternate = {"distributionInfo"}) private DistributionInfo defaultDistributionInfo; // all info about temporary partitions are save in "tempPartitions" @Getter - @SerializedName(value = "tempPartitions") + @SerializedName(value = "tps", alternate = {"tempPartitions"}) private TempPartitions tempPartitions = new TempPartitions(); // bloom filter columns - @SerializedName(value = "bfColumns") + @SerializedName(value = "bfc", alternate = {"bfColumns"}) private Set bfColumns; @SerializedName(value = "bfFpp") private double bfFpp; - @SerializedName(value = "colocateGroup") + @SerializedName(value = "cgs", alternate = "colocateGroup") private String colocateGroup; private boolean hasSequenceCol; @@ -187,10 +187,10 @@ public enum OlapTableState { // So we add this 'baseIndexId' to explicitly specify the base index id, // which should be different with table id. // The init value is -1, which means there is not partition and index at all. - @SerializedName(value = "baseIndexId") + @SerializedName(value = "bid", alternate = {"baseIndexId"}) private long baseIndexId = -1; - @SerializedName(value = "tableProperty") + @SerializedName(value = "tp", alternate = {"tableProperty"}) private TableProperty tableProperty; @SerializedName(value = "aIncg") diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java index 22813ef51185c7..0fcbef007437aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java @@ -88,37 +88,37 @@ public static class ReplicaContext { @SerializedName(value = "id") private long id; - @SerializedName(value = "backendId") + @SerializedName(value = "bid", alternate = {"backendId"}) private long backendId; // the version could be queried - @SerializedName(value = "version") + @SerializedName(value = "v", alternate = {"version"}) private volatile long version; @Deprecated - @SerializedName(value = "versionHash") + @SerializedName(value = "vh", alternate = {"versionHash"}) private long versionHash = 0L; private int schemaHash = -1; - @SerializedName(value = "dataSize") + @SerializedName(value = "ds", alternate = {"dataSize"}) private volatile long dataSize = 0; - @SerializedName(value = "remoteDataSize") + @SerializedName(value = "rds", alternate = {"remoteDataSize"}) private volatile long remoteDataSize = 0; - @SerializedName(value = "rowCount") + @SerializedName(value = "rc", alternate = {"rowCount"}) private volatile long rowCount = 0; - @SerializedName(value = "state") + @SerializedName(value = "st", alternate = {"state"}) private volatile ReplicaState state; // the last load failed version - @SerializedName(value = "lastFailedVersion") + @SerializedName(value = "lfv", alternate = {"lastFailedVersion"}) private long lastFailedVersion = -1L; @Deprecated - @SerializedName(value = "lastFailedVersionHash") + @SerializedName(value = "lfvh", alternate = {"lastFailedVersionHash"}) private long lastFailedVersionHash = 0L; // not serialized, not very important private long lastFailedTimestamp = 0; // the last load successful version - @SerializedName(value = "lastSuccessVersion") + @SerializedName(value = "lsv", alternate = {"lastSuccessVersion"}) private long lastSuccessVersion = -1L; @Deprecated - @SerializedName(value = "lastSuccessVersionHash") + @SerializedName(value = "lsvh", alternate = {"lastSuccessVersionHash"}) private long lastSuccessVersionHash = 0L; private volatile long totalVersionCount = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index 899eb94b8d0956..3266d0eeb660bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -83,20 +83,20 @@ public enum TabletStatus { @SerializedName(value = "id") protected long id; - @SerializedName(value = "replicas") + @SerializedName(value = "rs", alternate = {"replicas"}) protected List replicas; - @SerializedName(value = "checkedVersion") + @SerializedName(value = "cv", alternate = {"checkedVersion"}) private long checkedVersion; @Deprecated - @SerializedName(value = "checkedVersionHash") + @SerializedName(value = "cvs", alternate = {"checkedVersionHash"}) private long checkedVersionHash; - @SerializedName(value = "isConsistent") + @SerializedName(value = "ic", alternate = {"isConsistent"}) private boolean isConsistent; // cooldown conf - @SerializedName(value = "cooldownReplicaId") + @SerializedName(value = "cri", alternate = {"cooldownReplicaId"}) private long cooldownReplicaId = -1; - @SerializedName(value = "cooldownTerm") + @SerializedName(value = "ctm", alternate = {"cooldownTerm"}) private long cooldownTerm = -1; private ReentrantReadWriteLock cooldownConfLock = new ReentrantReadWriteLock();