[fix](meta) serialize expr in PartitionKey via gson

fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java

@@ -23,11 +23,13 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
 import com.google.common.collect.Lists;
+import com.google.gson.annotations.SerializedName;
 
 import java.util.ArrayList;
 import java.util.List;
 
 public enum PrimitiveType {
+ // DO NOT CHANGE desc and to string of these types, they are used in persist
  INVALID_TYPE("INVALID_TYPE", -1, TPrimitiveType.INVALID_TYPE, false),
  UNSUPPORTED("UNSUPPORTED_TYPE", -1, TPrimitiveType.UNSUPPORTED, false),
  // NULL_TYPE - used only in LiteralPredicate and NullLiteral to make NULLs compatible
@@ -699,10 +701,15 @@ public static boolean isImplicitCast(PrimitiveType type, PrimitiveType target) {
  return implicitCastMap.get(type).contains(target);
  }
 
+ @SerializedName("d")
  private final String description;
+ @SerializedName("s")
  private final int slotSize; // size of tuple slot for this type
+ @SerializedName("t")
  private final TPrimitiveType thriftType;
+ @SerializedName("a")
  private final boolean availableInDdl;
+ @SerializedName("it")
  private boolean isTimeType = false;
 
  PrimitiveType(String description, int slotSize, TPrimitiveType thriftType, boolean availableInDdl) {

fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java

@@ -101,8 +101,11 @@ public final class FeMetaVersion {
 
  public static final int VERSION_139 = 139;
 
+ public static final int VERSION_140 = 140;
+
  // note: when increment meta version, should assign the latest version to VERSION_CURRENT
- public static final int VERSION_CURRENT = VERSION_139;
+ public static final int VERSION_CURRENT = VERSION_140;
+
 
  // all logs meta version should >= the minimum version, so that we could remove many if clause, for example
  // if (FE_METAVERSION < VERSION_94) ...

fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java

@@ -51,17 +51,17 @@
 public class Index implements Writable {
  public static final int INDEX_ID_INIT_VALUE = -1;
 
- @SerializedName(value = "indexId")
+ @SerializedName(value = "i", alternate = {"indexId"})
  private long indexId = -1; // -1 for compatibale
- @SerializedName(value = "indexName")
+ @SerializedName(value = "in", alternate = {"indexName"})
  private String indexName;
- @SerializedName(value = "columns")
+ @SerializedName(value = "c", alternate = {"columns"})
  private List<String> columns;
- @SerializedName(value = "indexType")
+ @SerializedName(value = "it", alternate = {"indexType"})
  private IndexDef.IndexType indexType;
- @SerializedName(value = "properties")
+ @SerializedName(value = "pt", alternate = {"properties"})
  private Map<String, String> properties;
- @SerializedName(value = "comment")
+ @SerializedName(value = "ct", alternate = {"comment"})
  private String comment;
 
  public Index(long indexId, String indexName, List<String> columns,

fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java

@@ -49,26 +49,26 @@
 import java.util.stream.Collectors;
 
 public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
- @SerializedName(value = "indexId")
+ @SerializedName(value = "id", alternate = {"indexId"})
  private long indexId;
- @SerializedName(value = "schema")
+ @SerializedName(value = "sc", alternate = {"schema"})
  private List<Column> schema = Lists.newArrayList();
- @SerializedName(value = "schemaVersion")
+ @SerializedName(value = "sv", alternate = {"schemaVersion"})
  private int schemaVersion;
- @SerializedName(value = "schemaHash")
+ @SerializedName(value = "sh", alternate = {"schemaHash"})
  private int schemaHash;
- @SerializedName(value = "shortKeyColumnCount")
+ @SerializedName(value = "skcc", alternate = {"shortKeyColumnCount"})
  private short shortKeyColumnCount;
- @SerializedName(value = "storageType")
+ @SerializedName(value = "st", alternate = {"storageType"})
  private TStorageType storageType;
- @SerializedName(value = "keysType")
+ @SerializedName(value = "kt", alternate = {"keysType"})
  private KeysType keysType;
- @SerializedName(value = "defineStmt")
+ @SerializedName(value = "dst", alternate = {"defineStmt"})
  private OriginStatement defineStmt;
  //for light schema change
- @SerializedName(value = "maxColUniqueId")
+ @SerializedName(value = "mcui", alternate = {"maxColUniqueId"})
  private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
- @SerializedName(value = "indexes")
+ @SerializedName(value = "idx", alternate = {"indexes"})
  private List<Index> indexes;
 
  private Expr whereClause;

fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java

@@ -60,24 +60,24 @@ public enum PartitionState {
 
  @SerializedName(value = "id")
  private long id;
- @SerializedName(value = "name")
+ @SerializedName(value = "nm", alternate = {"name"})
  private String name;
- @SerializedName(value = "state")
+ @SerializedName(value = "st", alternate = {"state"})
  private PartitionState state;
- @SerializedName(value = "baseIndex")
+ @SerializedName(value = "bi", alternate = {"baseIndex"})
  private MaterializedIndex baseIndex;
  /**
  * Visible rollup indexes are indexes which are visible to user.
  * User can do query on them, show them in related 'show' stmt.
  */
- @SerializedName(value = "idToVisibleRollupIndex")
+ @SerializedName(value = "ivr", alternate = {"idToVisibleRollupIndex"})
  private Map<Long, MaterializedIndex> idToVisibleRollupIndex = Maps.newHashMap();
  /**
  * Shadow indexes are indexes which are not visible to user.
  * Query will not run on these shadow indexes, and user can not see them neither.
  * But load process will load data into these shadow indexes.
  */
- @SerializedName(value = "idToShadowIndex")
+ @SerializedName(value = "isi", alternate = {"idToShadowIndex"})
  private Map<Long, MaterializedIndex> idToShadowIndex = Maps.newHashMap();
 
  /**
@@ -88,21 +88,21 @@ public enum PartitionState {
 
  // not have committedVersion because committedVersion = nextVersion - 1
  @Deprecated
- @SerializedName(value = "committedVersionHash")
+ @SerializedName(value = "cvh", alternate = {"committedVersionHash"})
  private long committedVersionHash;
- @SerializedName(value = "visibleVersion")
+ @SerializedName(value = "vv", alternate = {"visibleVersion"})
  private long visibleVersion;
- @SerializedName(value = "visibleVersionTime")
+ @SerializedName(value = "vvt", alternate = {"visibleVersionTime"})
  private long visibleVersionTime;
  @Deprecated
- @SerializedName(value = "visibleVersionHash")
+ @SerializedName(value = "vvh", alternate = {"visibleVersionHash"})
  private long visibleVersionHash;
- @SerializedName(value = "nextVersion")
+ @SerializedName(value = "nv", alternate = {"nextVersion"})
  protected long nextVersion;
  @Deprecated
- @SerializedName(value = "nextVersionHash")
+ @SerializedName(value = "nvh", alternate = {"nextVersionHash"})
  private long nextVersionHash;
- @SerializedName(value = "distributionInfo")
+ @SerializedName(value = "di", alternate = {"distributionInfo"})
  private DistributionInfo distributionInfo;
 
  protected Partition() {

fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java

@@ -45,7 +45,6 @@
 import com.google.gson.JsonDeserializer;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
 import com.google.gson.JsonSerializationContext;
 import com.google.gson.JsonSerializer;
 import com.google.gson.annotations.SerializedName;
@@ -64,7 +63,6 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
  private static final Logger LOG = LogManager.getLogger(PartitionKey.class);
  @SerializedName("ks")
  private List<LiteralExpr> keys;
- @SerializedName("hk")
  private List<String> originHiveKeys;
  @SerializedName("ts")
  private List<PrimitiveType> types;
@@ -511,81 +509,50 @@ public int hashCode() {
  // added by ccr, and we have to follow.
  public static class PartitionKeySerializer
  implements JsonSerializer<PartitionKey>, JsonDeserializer<PartitionKey> {
+
  @Override
  public JsonElement serialize(PartitionKey partitionKey, java.lang.reflect.Type reflectType,
  JsonSerializationContext context) {
- JsonArray result = new JsonArray();
-
+ // for compatibility
  List<PrimitiveType> types = partitionKey.getTypes();
  List<LiteralExpr> keys = partitionKey.getKeys();
  int count = keys.size();
  if (count != types.size()) {
  throw new JsonParseException("Size of keys and types are not equal");
  }
 
+ JsonArray jsonArray = new JsonArray();
  for (int i = 0; i < count; i++) {
  JsonArray typeAndKey = new JsonArray();
- PrimitiveType type = types.get(i);
- if (keys.get(i).isNullLiteral()) {
- // save NULL_TYPE as type and real type as key
- typeAndKey.add(new JsonPrimitive(PrimitiveType.NULL_TYPE.toString()));
- typeAndKey.add(new JsonPrimitive(type.toString()));
- } else {
- typeAndKey.add(new JsonPrimitive(type.toString()));
-
- if (keys.get(i) == MaxLiteral.MAX_VALUE) {
- typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
- } else {
- switch (type) {
- case TINYINT:
- case SMALLINT:
- case INT:
- case BIGINT: {
- IntLiteral key = (IntLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.getLongValue()));
- }
- break;
- case LARGEINT: {
- LargeIntLiteral key = (LargeIntLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.getRealValue().toString()));
- }
- break;
- case DATE:
- case DATETIME:
- case DATEV2:
- case DATETIMEV2: {
- DateLiteral key = (DateLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.convertToString(type)));
- }
- break;
- case CHAR:
- case VARCHAR:
- case STRING: {
- StringLiteral key = (StringLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.getValue()));
- }
- break;
- case BOOLEAN: {
- BoolLiteral key = (BoolLiteral) keys.get(i);
- typeAndKey.add(new JsonPrimitive(key.getValue()));
- }
- break;
- default:
- throw new JsonParseException(
- "type[" + type.name() + "] not supported: ");
- }
- }
- }
-
- result.add(typeAndKey);
+ typeAndKey.add(context.serialize(types.get(i)));
+ typeAndKey.add(context.serialize(keys.get(i)));
+ jsonArray.add(typeAndKey);
  }
-
- return result;
+ return jsonArray;
  }
 
  @Override
  public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT,
  JsonDeserializationContext context) throws JsonParseException {
+ if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_140) {
+ return deserializeOld(json, typeOfT, context);
+ } else {
+ PartitionKey partitionKey = new PartitionKey();
+
+ JsonArray jsonArray = json.getAsJsonArray();
+ for (int i = 0; i < jsonArray.size(); i++) {
+ partitionKey.types.add(context.deserialize(jsonArray.get(i).getAsJsonArray().get(0),
+ PrimitiveType.class));
+ partitionKey.keys.add(context.deserialize(jsonArray.get(i).getAsJsonArray().get(1),
+ Expr.class));
+ }
+ return partitionKey;
+ }
+ }
+
+ // can be removed after 3.0.0
+ private PartitionKey deserializeOld(JsonElement json, java.lang.reflect.Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
  PartitionKey partitionKey = new PartitionKey();
  JsonArray jsonArray = json.getAsJsonArray();
  for (int i = 0; i < jsonArray.size(); i++) {
@@ -627,7 +594,7 @@ public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT
  case DATETIMEV2: {
  String value = typeAndKey.get(1).getAsString();
  try {
- literal = new DateLiteral(value, type);
+ literal = new DateLiteral(value, Type.fromPrimitiveType(type));
  } catch (AnalysisException e) {
  throw new JsonParseException("DateLiteral deserialize failed: " + e.getMessage());
  }