From 31270f553185c06048e80d04c3480a1a76918d66 Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <dataroaring@gmail.com>
Date: Wed, 19 Jun 2024 20:30:51 +0800
Subject: [PATCH] fix olap table

---
 .../org/apache/doris/catalog/OlapTable.java   | 65 +++++++++++++++++--
 1 file changed, 60 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 34317073c22ffaa..1c61e7cbc141f4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -54,6 +54,7 @@
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.mtmv.MTMVSnapshotIf;
 import org.apache.doris.mtmv.MTMVVersionSnapshot;
+import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.OriginStatement;
@@ -95,6 +96,7 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -112,7 +114,7 @@
  * Internal representation of tableFamilyGroup-related metadata. A OlaptableFamilyGroup contains several tableFamily.
  * Note: when you add a new olap table property, you should modify TableProperty class
  */
-public class OlapTable extends Table implements MTMVRelatedTableIf {
+public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProcessable {
     private static final Logger LOG = LogManager.getLogger(OlapTable.class);
 
     public enum OlapTableState {
@@ -150,6 +152,7 @@ public enum OlapTableState {
     @SerializedName("idToPartition")
     @Getter
     private Map<Long, Partition> idToPartition = new HashMap<>();
+    // handled in postgsonprocess
     @Getter
     private Map<String, Partition> nameToPartition = Maps.newTreeMap();
 
@@ -163,12 +166,12 @@ public enum OlapTableState {
 
     // bloom filter columns
     @SerializedName(value = "bfColumns")
-
     private Set<String> bfColumns;
+
     @SerializedName(value = "bfFpp")
     private double bfFpp;
-    @SerializedName(value = "colocateGroup")
 
+    @SerializedName(value = "colocateGroup")
     private String colocateGroup;
 
     private boolean hasSequenceCol;
@@ -189,7 +192,7 @@ public enum OlapTableState {
     @SerializedName(value = "tableProperty")
     private TableProperty tableProperty;
 
-    @SerializedName(value = "aig")
+    @SerializedName(value = "aIncg")
     private AutoIncrementGenerator autoIncrementGenerator;
 
     private volatile Statistics statistics = new Statistics();
@@ -1524,7 +1527,11 @@ public boolean isPartitionDistributed() {
         return false;
     }
 
-    @Deprecated
+    @Override
+    public void write(DataOutput out) throws IOException {
+        Text.writeString(out, GsonUtils.GSON.toJson(this));
+    }
+
     @Override
     public void readFields(DataInput in) throws IOException {
         super.readFields(in);
@@ -1556,6 +1563,7 @@ public void readFields(DataInput in) throws IOException {
         // partition and distribution info
         keysType = KeysType.valueOf(Text.readString(in));
 
+        // useless code see pr 3036
         // add the correct keys type in tmp index meta
         for (MaterializedIndexMeta indexMeta : tmpIndexMetaList) {
             indexMeta.setKeysType(keysType);
@@ -1644,6 +1652,53 @@ public void readFields(DataInput in) throws IOException {
         rebuildFullSchema();
     }
 
+    @Override
+    public void gsonPostProcess() throws IOException {
+
+        // HACK: the index id in MaterializedIndexMeta is not equals to the index id
+        // saved in OlapTable, because the table restore from snapshot is not reset
+        // the MaterializedIndexMeta correctly.
+        // for each index, reset the index id in MaterializedIndexMeta
+        for (Map.Entry<Long, MaterializedIndexMeta> entry : indexIdToMeta.entrySet()) {
+            long indexId = entry.getKey();
+            MaterializedIndexMeta indexMeta = entry.getValue();
+            if (indexMeta.getIndexId() != indexId) {
+                LOG.warn("HACK: the index id {} in materialized index meta of {} is not equals"
+                        + " to the index saved in table {} ({}), reset it to {}",
+                        indexMeta.getIndexId(), indexNameToId.get(indexId), name, id, indexId);
+                indexMeta.resetIndexIdForRestore(indexId);
+            }
+        }
+
+        // for each idToPartition, add partition to nameToPartition
+        for (Partition partition : idToPartition.values()) {
+            nameToPartition.put(partition.getName(), partition);
+        }
+
+        if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_124
+                    && autoIncrementGenerator != null) {
+            autoIncrementGenerator.setEditLog(Env.getCurrentEnv().getEditLog());
+        }
+        if (isAutoBucket()) {
+            defaultDistributionInfo.markAutoBucket();
+        }
+        RangePartitionInfo tempRangeInfo = tempPartitions.getPartitionInfo();
+        if (tempRangeInfo != null) {
+            for (long partitionId : tempRangeInfo.getIdToItem(false).keySet()) {
+                this.partitionInfo.addPartition(partitionId, true,
+                        tempRangeInfo.getItem(partitionId), tempRangeInfo.getDataProperty(partitionId),
+                        tempRangeInfo.getReplicaAllocation(partitionId), tempRangeInfo.getIsInMemory(partitionId),
+                        tempRangeInfo.getIsMutable(partitionId));
+            }
+        }
+        tempPartitions.unsetPartitionInfo();
+
+        // In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV.
+        // After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns.
+        // So, here we need to rebuild the fullSchema to ensure the correctness of the properties.
+        rebuildFullSchema();
+    }
+
     public OlapTable selectiveCopy(Collection<String> reservedPartitions, IndexExtState extState, boolean isForBackup) {
         OlapTable copied = DeepCopy.copy(this, OlapTable.class, FeConstants.meta_version);
         if (copied == null) {