fix olap table

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -54,6 +54,7 @@
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.mtmv.MTMVSnapshotIf;
 import org.apache.doris.mtmv.MTMVVersionSnapshot;
+import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.OriginStatement;
@@ -95,6 +96,7 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -112,7 +114,7 @@
  * Internal representation of tableFamilyGroup-related metadata. A OlaptableFamilyGroup contains several tableFamily.
  * Note: when you add a new olap table property, you should modify TableProperty class
  */
-public class OlapTable extends Table implements MTMVRelatedTableIf {
+public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProcessable {
  private static final Logger LOG = LogManager.getLogger(OlapTable.class);
 
  public enum OlapTableState {
@@ -150,6 +152,7 @@ public enum OlapTableState {
  @SerializedName("idToPartition")
  @Getter
  private Map<Long, Partition> idToPartition = new HashMap<>();
+ // handled in postgsonprocess
  @Getter
  private Map<String, Partition> nameToPartition = Maps.newTreeMap();
 
@@ -163,12 +166,12 @@ public enum OlapTableState {
 
  // bloom filter columns
  @SerializedName(value = "bfColumns")
-
  private Set<String> bfColumns;
+
  @SerializedName(value = "bfFpp")
  private double bfFpp;
- @SerializedName(value = "colocateGroup")
 
+ @SerializedName(value = "colocateGroup")
  private String colocateGroup;
 
  private boolean hasSequenceCol;
@@ -189,7 +192,7 @@ public enum OlapTableState {
  @SerializedName(value = "tableProperty")
  private TableProperty tableProperty;
 
- @SerializedName(value = "aig")
+ @SerializedName(value = "aIncg")
  private AutoIncrementGenerator autoIncrementGenerator;
 
  private volatile Statistics statistics = new Statistics();
@@ -1540,7 +1543,11 @@ public boolean isPartitionDistributed() {
  return false;
  }
 
- @Deprecated
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
+ }
+
  @Override
  public void readFields(DataInput in) throws IOException {
  super.readFields(in);
@@ -1572,6 +1579,7 @@ public void readFields(DataInput in) throws IOException {
  // partition and distribution info
  keysType = KeysType.valueOf(Text.readString(in));
 
+ // useless code see pr 3036
  // add the correct keys type in tmp index meta
  for (MaterializedIndexMeta indexMeta : tmpIndexMetaList) {
  indexMeta.setKeysType(keysType);
@@ -1660,6 +1668,53 @@ public void readFields(DataInput in) throws IOException {
  rebuildFullSchema();
  }
 
+ @Override
+ public void gsonPostProcess() throws IOException {
+
+ // HACK: the index id in MaterializedIndexMeta is not equals to the index id
+ // saved in OlapTable, because the table restore from snapshot is not reset
+ // the MaterializedIndexMeta correctly.
+ // for each index, reset the index id in MaterializedIndexMeta
+ for (Map.Entry<Long, MaterializedIndexMeta> entry : indexIdToMeta.entrySet()) {
+ long indexId = entry.getKey();
+ MaterializedIndexMeta indexMeta = entry.getValue();
+ if (indexMeta.getIndexId() != indexId) {
+ LOG.warn("HACK: the index id {} in materialized index meta of {} is not equals"
+ + " to the index saved in table {} ({}), reset it to {}",
+ indexMeta.getIndexId(), indexNameToId.get(indexId), name, id, indexId);
+ indexMeta.resetIndexIdForRestore(indexId);
+ }
+ }
+
+ // for each idToPartition, add partition to nameToPartition
+ for (Partition partition : idToPartition.values()) {
+ nameToPartition.put(partition.getName(), partition);
+ }
+
+ if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_124
+ && autoIncrementGenerator != null) {
+ autoIncrementGenerator.setEditLog(Env.getCurrentEnv().getEditLog());
+ }
+ if (isAutoBucket()) {
+ defaultDistributionInfo.markAutoBucket();
+ }
+ RangePartitionInfo tempRangeInfo = tempPartitions.getPartitionInfo();
+ if (tempRangeInfo != null) {
+ for (long partitionId : tempRangeInfo.getIdToItem(false).keySet()) {
+ this.partitionInfo.addPartition(partitionId, true,
+ tempRangeInfo.getItem(partitionId), tempRangeInfo.getDataProperty(partitionId),
+ tempRangeInfo.getReplicaAllocation(partitionId), tempRangeInfo.getIsInMemory(partitionId),
+ tempRangeInfo.getIsMutable(partitionId));
+ }
+ }
+ tempPartitions.unsetPartitionInfo();
+
+ // In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV.
+ // After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns.
+ // So, here we need to rebuild the fullSchema to ensure the correctness of the properties.
+ rebuildFullSchema();
+ }
+
  public OlapTable selectiveCopy(Collection<String> reservedPartitions, IndexExtState extState, boolean isForBackup) {
  OlapTable copied = DeepCopy.copy(this, OlapTable.class, FeConstants.meta_version);
  if (copied == null) {