From cdb1d1de6c7f8b0e5334d8fe070aa53e6b34968a Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Thu, 4 Jul 2024 17:41:04 +0800 Subject: [PATCH] [fix](meta) serialize expr in PartitionKey via gson --- .../apache/doris/catalog/PrimitiveType.java | 7 ++ .../apache/doris/common/FeMetaVersion.java | 5 +- .../java/org/apache/doris/catalog/Index.java | 12 +-- .../doris/catalog/MaterializedIndexMeta.java | 20 ++--- .../org/apache/doris/catalog/Partition.java | 24 ++--- .../apache/doris/catalog/PartitionKey.java | 87 ++++++------------- 6 files changed, 66 insertions(+), 89 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 50cc6c4c1c0b65..e5b1b9c958b1a0 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -23,11 +23,13 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.util.ArrayList; import java.util.List; public enum PrimitiveType { + // DO NOT CHANGE desc and to string of these types, they are used in persist INVALID_TYPE("INVALID_TYPE", -1, TPrimitiveType.INVALID_TYPE, false), UNSUPPORTED("UNSUPPORTED_TYPE", -1, TPrimitiveType.UNSUPPORTED, false), // NULL_TYPE - used only in LiteralPredicate and NullLiteral to make NULLs compatible @@ -699,10 +701,15 @@ public static boolean isImplicitCast(PrimitiveType type, PrimitiveType target) { return implicitCastMap.get(type).contains(target); } + @SerializedName("d") private final String description; + @SerializedName("s") private final int slotSize; // size of tuple slot for this type + @SerializedName("t") private final TPrimitiveType thriftType; + @SerializedName("a") private final boolean availableInDdl; + @SerializedName("it") private boolean isTimeType = false; PrimitiveType(String description, int slotSize, TPrimitiveType thriftType, boolean availableInDdl) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 4f2be870987aea..a81a01227e75f8 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -101,8 +101,11 @@ public final class FeMetaVersion { public static final int VERSION_139 = 139; + public static final int VERSION_140 = 140; + // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_139; + public static final int VERSION_CURRENT = VERSION_140; + // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 5b0c9976819f02..52290bc7479277 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -51,17 +51,17 @@ public class Index implements Writable { public static final int INDEX_ID_INIT_VALUE = -1; - @SerializedName(value = "indexId") + @SerializedName(value = "i", alternate = {"indexId"}) private long indexId = -1; // -1 for compatibale - @SerializedName(value = "indexName") + @SerializedName(value = "in", alternate = {"indexName"}) private String indexName; - @SerializedName(value = "columns") + @SerializedName(value = "c", alternate = {"columns"}) private List columns; - @SerializedName(value = "indexType") + @SerializedName(value = "it", alternate = {"indexType"}) private IndexDef.IndexType indexType; - @SerializedName(value = "properties") + @SerializedName(value = "pt", alternate = {"properties"}) private Map properties; - @SerializedName(value = "comment") + @SerializedName(value = "ct", alternate = {"comment"}) private String comment; public Index(long indexId, String indexName, List columns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 79274b3bb0bbd6..0069d4e4e02798 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -49,26 +49,26 @@ import java.util.stream.Collectors; public class MaterializedIndexMeta implements Writable, GsonPostProcessable { - @SerializedName(value = "indexId") + @SerializedName(value = "id", alternate = {"indexId"}) private long indexId; - @SerializedName(value = "schema") + @SerializedName(value = "sc", alternate = {"schema"}) private List schema = Lists.newArrayList(); - @SerializedName(value = "schemaVersion") + @SerializedName(value = "sv", alternate = {"schemaVersion"}) private int schemaVersion; - @SerializedName(value = "schemaHash") + @SerializedName(value = "sh", alternate = {"schemaHash"}) private int schemaHash; - @SerializedName(value = "shortKeyColumnCount") + @SerializedName(value = "skcc", alternate = {"shortKeyColumnCount"}) private short shortKeyColumnCount; - @SerializedName(value = "storageType") + @SerializedName(value = "st", alternate = {"storageType"}) private TStorageType storageType; - @SerializedName(value = "keysType") + @SerializedName(value = "kt", alternate = {"keysType"}) private KeysType keysType; - @SerializedName(value = "defineStmt") + @SerializedName(value = "dst", alternate = {"defineStmt"}) private OriginStatement defineStmt; //for light schema change - @SerializedName(value = "maxColUniqueId") + @SerializedName(value = "mcui", alternate = {"maxColUniqueId"}) private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE; - @SerializedName(value = "indexes") + @SerializedName(value = "idx", alternate = {"indexes"}) private List indexes; private Expr whereClause; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java index ef94da902b874b..c28d44a552138c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java @@ -60,24 +60,24 @@ public enum PartitionState { @SerializedName(value = "id") private long id; - @SerializedName(value = "name") + @SerializedName(value = "nm", alternate = {"name"}) private String name; - @SerializedName(value = "state") + @SerializedName(value = "st", alternate = {"state"}) private PartitionState state; - @SerializedName(value = "baseIndex") + @SerializedName(value = "bi", alternate = {"baseIndex"}) private MaterializedIndex baseIndex; /** * Visible rollup indexes are indexes which are visible to user. * User can do query on them, show them in related 'show' stmt. */ - @SerializedName(value = "idToVisibleRollupIndex") + @SerializedName(value = "ivr", alternate = {"idToVisibleRollupIndex"}) private Map idToVisibleRollupIndex = Maps.newHashMap(); /** * Shadow indexes are indexes which are not visible to user. * Query will not run on these shadow indexes, and user can not see them neither. * But load process will load data into these shadow indexes. */ - @SerializedName(value = "idToShadowIndex") + @SerializedName(value = "isi", alternate = {"idToShadowIndex"}) private Map idToShadowIndex = Maps.newHashMap(); /** @@ -88,21 +88,21 @@ public enum PartitionState { // not have committedVersion because committedVersion = nextVersion - 1 @Deprecated - @SerializedName(value = "committedVersionHash") + @SerializedName(value = "cvh", alternate = {"committedVersionHash"}) private long committedVersionHash; - @SerializedName(value = "visibleVersion") + @SerializedName(value = "vv", alternate = {"visibleVersion"}) private long visibleVersion; - @SerializedName(value = "visibleVersionTime") + @SerializedName(value = "vvt", alternate = {"visibleVersionTime"}) private long visibleVersionTime; @Deprecated - @SerializedName(value = "visibleVersionHash") + @SerializedName(value = "vvh", alternate = {"visibleVersionHash"}) private long visibleVersionHash; - @SerializedName(value = "nextVersion") + @SerializedName(value = "nv", alternate = {"nextVersion"}) protected long nextVersion; @Deprecated - @SerializedName(value = "nextVersionHash") + @SerializedName(value = "nvh", alternate = {"nextVersionHash"}) private long nextVersionHash; - @SerializedName(value = "distributionInfo") + @SerializedName(value = "di", alternate = {"distributionInfo"}) private DistributionInfo distributionInfo; protected Partition() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 4d060880d8f7e9..2ba5b3509d1c60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -45,7 +45,6 @@ import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; import com.google.gson.annotations.SerializedName; @@ -64,7 +63,6 @@ public class PartitionKey implements Comparable, Writable { private static final Logger LOG = LogManager.getLogger(PartitionKey.class); @SerializedName("ks") private List keys; - @SerializedName("hk") private List originHiveKeys; @SerializedName("ts") private List types; @@ -511,11 +509,11 @@ public int hashCode() { // added by ccr, and we have to follow. public static class PartitionKeySerializer implements JsonSerializer, JsonDeserializer { + @Override public JsonElement serialize(PartitionKey partitionKey, java.lang.reflect.Type reflectType, JsonSerializationContext context) { - JsonArray result = new JsonArray(); - + // for compatibility List types = partitionKey.getTypes(); List keys = partitionKey.getKeys(); int count = keys.size(); @@ -523,69 +521,38 @@ public JsonElement serialize(PartitionKey partitionKey, java.lang.reflect.Type r throw new JsonParseException("Size of keys and types are not equal"); } + JsonArray jsonArray = new JsonArray(); for (int i = 0; i < count; i++) { JsonArray typeAndKey = new JsonArray(); - PrimitiveType type = types.get(i); - if (keys.get(i).isNullLiteral()) { - // save NULL_TYPE as type and real type as key - typeAndKey.add(new JsonPrimitive(PrimitiveType.NULL_TYPE.toString())); - typeAndKey.add(new JsonPrimitive(type.toString())); - } else { - typeAndKey.add(new JsonPrimitive(type.toString())); - - if (keys.get(i) == MaxLiteral.MAX_VALUE) { - typeAndKey.add(new JsonPrimitive("MAX_VALUE")); - } else { - switch (type) { - case TINYINT: - case SMALLINT: - case INT: - case BIGINT: { - IntLiteral key = (IntLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getLongValue())); - } - break; - case LARGEINT: { - LargeIntLiteral key = (LargeIntLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getRealValue().toString())); - } - break; - case DATE: - case DATETIME: - case DATEV2: - case DATETIMEV2: { - DateLiteral key = (DateLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.convertToString(type))); - } - break; - case CHAR: - case VARCHAR: - case STRING: { - StringLiteral key = (StringLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getValue())); - } - break; - case BOOLEAN: { - BoolLiteral key = (BoolLiteral) keys.get(i); - typeAndKey.add(new JsonPrimitive(key.getValue())); - } - break; - default: - throw new JsonParseException( - "type[" + type.name() + "] not supported: "); - } - } - } - - result.add(typeAndKey); + typeAndKey.add(context.serialize(types.get(i))); + typeAndKey.add(context.serialize(keys.get(i))); + jsonArray.add(typeAndKey); } - - return result; + return jsonArray; } @Override public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_140) { + return deserializeOld(json, typeOfT, context); + } else { + PartitionKey partitionKey = new PartitionKey(); + + JsonArray jsonArray = json.getAsJsonArray(); + for (int i = 0; i < jsonArray.size(); i++) { + partitionKey.types.add(context.deserialize(jsonArray.get(i).getAsJsonArray().get(0), + PrimitiveType.class)); + partitionKey.keys.add(context.deserialize(jsonArray.get(i).getAsJsonArray().get(1), + Expr.class)); + } + return partitionKey; + } + } + + // can be removed after 3.0.0 + private PartitionKey deserializeOld(JsonElement json, java.lang.reflect.Type typeOfT, + JsonDeserializationContext context) throws JsonParseException { PartitionKey partitionKey = new PartitionKey(); JsonArray jsonArray = json.getAsJsonArray(); for (int i = 0; i < jsonArray.size(); i++) { @@ -627,7 +594,7 @@ public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT case DATETIMEV2: { String value = typeAndKey.get(1).getAsString(); try { - literal = new DateLiteral(value, type); + literal = new DateLiteral(value, Type.fromPrimitiveType(type)); } catch (AnalysisException e) { throw new JsonParseException("DateLiteral deserialize failed: " + e.getMessage()); }