[fix](meta) do not serialize nametotable map

Gson can not handle size beyonds 2GB.

fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java

@@ -97,8 +97,10 @@ public final class FeMetaVersion {
  // For mate gson
  public static final int VERSION_137 = 137;
 
+ public static final int VERSION_138 = 138;
+
  // note: when increment meta version, should assign the latest version to VERSION_CURRENT
- public static final int VERSION_CURRENT = VERSION_137;
+ public static final int VERSION_CURRENT = VERSION_138;
 
  // all logs meta version should >= the minimum version, so that we could remove many if clause, for example
  // if (FE_METAVERSION < VERSION_94) ...

fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java

@@ -91,7 +91,6 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>,
 
  // table family group map
  private final Map<Long, Table> idToTable;
- @SerializedName(value = "nameToTable")
  private ConcurrentMap<String, Table> nameToTable;
  // table name lower cast -> table name
  private final Map<String, String> lowerCaseToTableName;
@@ -595,8 +594,35 @@ public static Database read(DataInput in) throws IOException {
  Database db = new Database();
  db.readFields(in);
  return db;
- } else {
+ } else if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_138) {
  return GsonUtils.GSON.fromJson(Text.readString(in), Database.class);
+ } else {
+ Database db = GsonUtils.GSON.fromJson(Text.readString(in), Database.class);
+ db.readTables(in);
+ return db;
+ }
+ }
+
+ private void writeTables(DataOutput out) throws IOException {
+ out.writeInt(nameToTable.size());
+ for (Table table : nameToTable.values()) {
+ Text.writeString(out, GsonUtils.GSON.toJson(table));
+ }
+ }
+
+ private void readTables(DataInput in) throws IOException {
+ nameToTable = Maps.newConcurrentMap();
+ int numTables = in.readInt();
+ for (int i = 0; i < numTables; ++i) {
+ Table table = Table.read(in);
+ table.setQualifiedDbName(fullQualifiedName);
+ if (table instanceof MTMV) {
+ Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) table, id);
+ }
+ String tableName = table.getName();
+ nameToTable.put(tableName, table);
+ idToTable.put(table.getId(), table);
+ lowerCaseToTableName.put(tableName.toLowerCase(), tableName);
  }
  }
 
@@ -650,6 +676,10 @@ public String getSignature(int signatureVersion) {
  public void write(DataOutput out) throws IOException {
  discardHudiTable();
  Text.writeString(out, GsonUtils.GSON.toJson(this));
+ if (FeMetaVersion.VERSION_138 <= Env.getCurrentEnvJournalVersion()) {
+ writeTables(out);
+ return;
+ }
  }
 
 

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -135,44 +135,44 @@ public enum OlapTableState {
  WAITING_STABLE
  }
 
- @SerializedName(value = "state")
+ @SerializedName(value = "s", alternate = {"state"})
  private volatile OlapTableState state;
 
  // index id -> index meta
- @SerializedName("indexIdToMeta")
+ @SerializedName(value = "itm", alternate = {"indexIdToMeta"})
  private Map<Long, MaterializedIndexMeta> indexIdToMeta = Maps.newHashMap();
  // index name -> index id
- @SerializedName("indexNameToId")
+ @SerializedName(value = "inti", alternate = {"indexNameToId"})
  private Map<String, Long> indexNameToId = Maps.newHashMap();
 
- @SerializedName("keysType")
+ @SerializedName(value = "kt", alternate = {"keysType"})
  private KeysType keysType;
  @Setter
- @SerializedName("partitionInfo")
+ @SerializedName(value = "pi", alternate = {"partitionInfo"})
  private PartitionInfo partitionInfo;
- @SerializedName("idToPartition")
+ @SerializedName(value = "itp", alternate = {"idToPartition"})
  @Getter
  private ConcurrentHashMap<Long, Partition> idToPartition = new ConcurrentHashMap<>();
  // handled in postgsonprocess
  @Getter
  private Map<String, Partition> nameToPartition = Maps.newTreeMap();
 
- @SerializedName(value = "distributionInfo")
+ @SerializedName(value = "di", alternate = {"distributionInfo"})
  private DistributionInfo defaultDistributionInfo;
 
  // all info about temporary partitions are save in "tempPartitions"
  @Getter
- @SerializedName(value = "tempPartitions")
+ @SerializedName(value = "tps", alternate = {"tempPartitions"})
  private TempPartitions tempPartitions = new TempPartitions();
 
  // bloom filter columns
- @SerializedName(value = "bfColumns")
+ @SerializedName(value = "bfc", alternate = {"bfColumns"})
  private Set<String> bfColumns;
 
  @SerializedName(value = "bfFpp")
  private double bfFpp;
 
- @SerializedName(value = "colocateGroup")
+ @SerializedName(value = "cgs", alternate = "colocateGroup")
  private String colocateGroup;
 
  private boolean hasSequenceCol;
@@ -187,10 +187,10 @@ public enum OlapTableState {
  // So we add this 'baseIndexId' to explicitly specify the base index id,
  // which should be different with table id.
  // The init value is -1, which means there is not partition and index at all.
- @SerializedName(value = "baseIndexId")
+ @SerializedName(value = "bid", alternate = {"baseIndexId"})
  private long baseIndexId = -1;
 
- @SerializedName(value = "tableProperty")
+ @SerializedName(value = "tp", alternate = {"tableProperty"})
  private TableProperty tableProperty;
 
  @SerializedName(value = "aIncg")

fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java

@@ -88,37 +88,37 @@ public static class ReplicaContext {
 
  @SerializedName(value = "id")
  private long id;
- @SerializedName(value = "backendId")
+ @SerializedName(value = "bid", alternate = {"backendId"})
  private long backendId;
  // the version could be queried
- @SerializedName(value = "version")
+ @SerializedName(value = "v", alternate = {"version"})
  private volatile long version;
  @Deprecated
- @SerializedName(value = "versionHash")
+ @SerializedName(value = "vh", alternate = {"versionHash"})
  private long versionHash = 0L;
  private int schemaHash = -1;
- @SerializedName(value = "dataSize")
+ @SerializedName(value = "ds", alternate = {"dataSize"})
  private volatile long dataSize = 0;
- @SerializedName(value = "remoteDataSize")
+ @SerializedName(value = "rds", alternate = {"remoteDataSize"})
  private volatile long remoteDataSize = 0;
- @SerializedName(value = "rowCount")
+ @SerializedName(value = "rc", alternate = {"rowCount"})
  private volatile long rowCount = 0;
- @SerializedName(value = "state")
+ @SerializedName(value = "st", alternate = {"state"})
  private volatile ReplicaState state;
 
  // the last load failed version
- @SerializedName(value = "lastFailedVersion")
+ @SerializedName(value = "lfv", alternate = {"lastFailedVersion"})
  private long lastFailedVersion = -1L;
  @Deprecated
- @SerializedName(value = "lastFailedVersionHash")
+ @SerializedName(value = "lfvh", alternate = {"lastFailedVersionHash"})
  private long lastFailedVersionHash = 0L;
  // not serialized, not very important
  private long lastFailedTimestamp = 0;
  // the last load successful version
- @SerializedName(value = "lastSuccessVersion")
+ @SerializedName(value = "lsv", alternate = {"lastSuccessVersion"})
  private long lastSuccessVersion = -1L;
  @Deprecated
- @SerializedName(value = "lastSuccessVersionHash")
+ @SerializedName(value = "lsvh", alternate = {"lastSuccessVersionHash"})
  private long lastSuccessVersionHash = 0L;
 
  private volatile long totalVersionCount = -1;

fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java

@@ -83,20 +83,20 @@ public enum TabletStatus {
 
  @SerializedName(value = "id")
  protected long id;
- @SerializedName(value = "replicas")
+ @SerializedName(value = "rs", alternate = {"replicas"})
  protected List<Replica> replicas;
- @SerializedName(value = "checkedVersion")
+ @SerializedName(value = "cv", alternate = {"checkedVersion"})
  private long checkedVersion;
  @Deprecated
- @SerializedName(value = "checkedVersionHash")
+ @SerializedName(value = "cvs", alternate = {"checkedVersionHash"})
  private long checkedVersionHash;
- @SerializedName(value = "isConsistent")
+ @SerializedName(value = "ic", alternate = {"isConsistent"})
  private boolean isConsistent;
 
  // cooldown conf
- @SerializedName(value = "cooldownReplicaId")
+ @SerializedName(value = "cri", alternate = {"cooldownReplicaId"})
  private long cooldownReplicaId = -1;
- @SerializedName(value = "cooldownTerm")
+ @SerializedName(value = "ctm", alternate = {"cooldownTerm"})
  private long cooldownTerm = -1;
  private ReentrantReadWriteLock cooldownConfLock = new ReentrantReadWriteLock();