diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 80c3aa4df279e01..5e5a4f849ff02a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -54,6 +54,7 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVSnapshotIf; import org.apache.doris.mtmv.MTMVVersionSnapshot; +import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; @@ -95,6 +96,7 @@ import org.apache.logging.log4j.Logger; import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -112,7 +114,7 @@ * Internal representation of tableFamilyGroup-related metadata. A OlaptableFamilyGroup contains several tableFamily. * Note: when you add a new olap table property, you should modify TableProperty class */ -public class OlapTable extends Table implements MTMVRelatedTableIf { +public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(OlapTable.class); public enum OlapTableState { @@ -150,6 +152,7 @@ public enum OlapTableState { @SerializedName("idToPartition") @Getter private Map idToPartition = new HashMap<>(); + // handled in postgsonprocess @Getter private Map nameToPartition = Maps.newTreeMap(); @@ -163,12 +166,12 @@ public enum OlapTableState { // bloom filter columns @SerializedName(value = "bfColumns") - private Set bfColumns; + @SerializedName(value = "bfFpp") private double bfFpp; - @SerializedName(value = "colocateGroup") + @SerializedName(value = "colocateGroup") private String colocateGroup; private boolean hasSequenceCol; @@ -189,7 +192,7 @@ public enum OlapTableState { @SerializedName(value = "tableProperty") private TableProperty tableProperty; - @SerializedName(value = "aig") + @SerializedName(value = "aIncg") private AutoIncrementGenerator autoIncrementGenerator; private volatile Statistics statistics = new Statistics(); @@ -1540,7 +1543,11 @@ public boolean isPartitionDistributed() { return false; } - @Deprecated + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); @@ -1572,6 +1579,7 @@ public void readFields(DataInput in) throws IOException { // partition and distribution info keysType = KeysType.valueOf(Text.readString(in)); + // useless code see pr 3036 // add the correct keys type in tmp index meta for (MaterializedIndexMeta indexMeta : tmpIndexMetaList) { indexMeta.setKeysType(keysType); @@ -1660,6 +1668,53 @@ public void readFields(DataInput in) throws IOException { rebuildFullSchema(); } + @Override + public void gsonPostProcess() throws IOException { + + // HACK: the index id in MaterializedIndexMeta is not equals to the index id + // saved in OlapTable, because the table restore from snapshot is not reset + // the MaterializedIndexMeta correctly. + // for each index, reset the index id in MaterializedIndexMeta + for (Map.Entry entry : indexIdToMeta.entrySet()) { + long indexId = entry.getKey(); + MaterializedIndexMeta indexMeta = entry.getValue(); + if (indexMeta.getIndexId() != indexId) { + LOG.warn("HACK: the index id {} in materialized index meta of {} is not equals" + + " to the index saved in table {} ({}), reset it to {}", + indexMeta.getIndexId(), indexNameToId.get(indexId), name, id, indexId); + indexMeta.resetIndexIdForRestore(indexId); + } + } + + // for each idToPartition, add partition to nameToPartition + for (Partition partition : idToPartition.values()) { + nameToPartition.put(partition.getName(), partition); + } + + if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_124 + && autoIncrementGenerator != null) { + autoIncrementGenerator.setEditLog(Env.getCurrentEnv().getEditLog()); + } + if (isAutoBucket()) { + defaultDistributionInfo.markAutoBucket(); + } + RangePartitionInfo tempRangeInfo = tempPartitions.getPartitionInfo(); + if (tempRangeInfo != null) { + for (long partitionId : tempRangeInfo.getIdToItem(false).keySet()) { + this.partitionInfo.addPartition(partitionId, true, + tempRangeInfo.getItem(partitionId), tempRangeInfo.getDataProperty(partitionId), + tempRangeInfo.getReplicaAllocation(partitionId), tempRangeInfo.getIsInMemory(partitionId), + tempRangeInfo.getIsMutable(partitionId)); + } + } + tempPartitions.unsetPartitionInfo(); + + // In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV. + // After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns. + // So, here we need to rebuild the fullSchema to ensure the correctness of the properties. + rebuildFullSchema(); + } + public OlapTable selectiveCopy(Collection reservedPartitions, IndexExtState extState, boolean isForBackup) { OlapTable copied = DeepCopy.copy(this, OlapTable.class, FeConstants.meta_version); if (copied == null) {