diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java index 8c32bafe5fafdd..191c5ba9a40ed5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java @@ -32,6 +32,7 @@ public class ShowCatalogStmt extends ShowStmt { .addColumn(new Column("Type", ScalarType.createStringType())) .addColumn(new Column("IsCurrent", ScalarType.createStringType())) .addColumn(new Column("CreateTime", ScalarType.createStringType())) + .addColumn(new Column("LastUpdateTime", ScalarType.createStringType())) .addColumn(new Column("Comment", ScalarType.createStringType())) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java index 7672b8b76d2006..f9c5d94601f787 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java @@ -235,4 +235,8 @@ default OlapTable getOlapTableOrAnalysisException(String tableName) throws Analy void dropTable(String tableName); CatalogIf getCatalog(); + + default long getLastUpdateTime() { + return -1L; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index e9d0e91c25fd94..78717f0eca769d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -232,5 +232,9 @@ default Partition getPartition(String name) { default boolean isManagedTable() { return getType() == TableType.OLAP || getType() == TableType.MATERIALIZED_VIEW; } + + default long getLastUpdateTime() { + return -1L; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java index e1b582a5403bcc..0a82d37ff3d5cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java @@ -73,6 +73,8 @@ public abstract class ExternalDatabase protected Map tableNameToId = Maps.newConcurrentMap(); @SerializedName(value = "idToTbl") protected Map idToTbl = Maps.newConcurrentMap(); + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; protected final InitDatabaseLog.Type dbLogType; protected ExternalCatalog extCatalog; protected boolean invalidCacheInInit = true; @@ -147,6 +149,7 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } tableNameToId = tmpTableNameToId; idToTbl = tmpIdToTbl; + lastUpdateTime = log.getLastUpdateTime(); initialized = true; } @@ -179,6 +182,10 @@ protected void init() { tableNameToId = tmpTableNameToId; idToTbl = tmpIdToTbl; } + + long currentTime = System.currentTimeMillis(); + lastUpdateTime = currentTime; + initDatabaseLog.setLastUpdateTime(lastUpdateTime); initialized = true; Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } @@ -307,6 +314,14 @@ public T getTableNullable(long tableId) { return idToTbl.get(tableId); } + public long getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(long lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java index 88ba9bc94e2804..f4c76cda7a98a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java @@ -69,6 +69,8 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long timestamp; @SerializedName(value = "dbName") protected String dbName; + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; protected boolean objectCreated; protected ExternalCatalog catalog; @@ -342,6 +344,11 @@ public Optional getColumnStatistic(String colName) { * * @return */ + public List initSchemaAndUpdateTime() { + lastUpdateTime = System.currentTimeMillis(); + return initSchema(); + } + public List initSchema() { throw new NotImplementedException("implement in sub class"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index 48e6b6ab46ad7e..854b4dddc796df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.ListComparator; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogIf; import com.google.common.base.Preconditions; @@ -37,7 +38,7 @@ */ public class CatalogsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("CatalogIds").add("CatalogName").add("DatabaseNum") + .add("CatalogIds").add("CatalogName").add("DatabaseNum").add("LastUpdateTime") .build(); private Env env; @@ -90,6 +91,7 @@ public ProcResult fetchResult() throws AnalysisException { catalogInfo.add(catalog.getId()); catalogInfo.add(catalog.getName()); catalogInfo.add(catalog.getDbNames().size()); + catalogInfo.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime())); catalogInfos.add(catalogInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java index 431c618a6f3301..a3818099d8e641 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java @@ -43,7 +43,7 @@ public class DbsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("DbId").add("DbName").add("TableNum").add("Size").add("Quota") .add("LastConsistencyCheckTime").add("ReplicaCount").add("ReplicaQuota") - .add("TransactionQuota") + .add("TransactionQuota").add("LastUpdateTime") .build(); private Env env; @@ -122,7 +122,7 @@ public ProcResult fetchResult() throws AnalysisException { dbInfo.add(replicaCount); dbInfo.add(replicaQuota); dbInfo.add(transactionQuota); - + dbInfo.add(TimeUtils.longToTimeString(db.getLastUpdateTime())); } finally { db.readUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java index 3b43ec887102cb..be51b157dedf6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java @@ -45,6 +45,7 @@ public class TablesProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("TableId").add("TableName").add("IndexNum").add("PartitionColumnName") .add("PartitionNum").add("State").add("Type").add("LastConsistencyCheckTime").add("ReplicaCount") + .add("LastUpdateTime") .build(); private DatabaseIf db; @@ -128,8 +129,7 @@ public ProcResult fetchResult() throws AnalysisException { tableInfo.add(FeConstants.null_string); tableInfo.add(replicaCount); } - - + tableInfo.add(TimeUtils.longToTimeString(table.getLastUpdateTime())); tableInfos.add(tableInfo); } finally { table.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 6fa054baa5f05c..b84c769eacff5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -153,6 +153,10 @@ default void onClose() { String getComment(); + default long getLastUpdateTime() { + return -1L; + } + default CatalogLog constructEditLog() { CatalogLog log = new CatalogLog(); log.setCatalogId(getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 0673e0e5fc2ec1..9d4e694e9a6662 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -43,6 +43,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.OperationType; import org.apache.doris.persist.gson.GsonPostProcessable; @@ -378,7 +379,6 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(), CaseSensibility.CATALOG.getCaseSensibility()); } - for (CatalogIf catalog : nameToCatalog.values()) { if (Env.getCurrentEnv().getAccessManager() .checkCtlPriv(ConnectContext.get(), catalog.getName(), PrivPredicate.SHOW)) { @@ -399,6 +399,7 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) Map props = catalog.getProperties(); String createTime = props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED"); row.add(createTime); + row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime())); row.add(catalog.getComment()); rows.add(row); } @@ -714,6 +715,7 @@ public void dropExternalTable(String dbName, String tableName, String catalogNam log.setCatalogId(catalog.getId()); log.setDbId(db.getId()); log.setTableId(table.getId()); + log.setLastUpdateTime(System.currentTimeMillis()); replayDropExternalTable(log); Env.getCurrentEnv().getEditLog().logDropExternalTable(log); } @@ -739,6 +741,7 @@ public void replayDropExternalTable(ExternalObjectLog log) { db.writeLock(); try { db.dropTable(table.getName()); + db.setLastUpdateTime(log.getLastUpdateTime()); } finally { db.writeUnlock(); } @@ -788,6 +791,7 @@ public void createExternalTableFromEvent(String dbName, String tableName, String log.setDbId(db.getId()); log.setTableName(tableName); log.setTableId(Env.getCurrentEnv().getNextId()); + log.setLastUpdateTime(System.currentTimeMillis()); replayCreateExternalTableFromEvent(log); Env.getCurrentEnv().getEditLog().logCreateExternalTable(log); } @@ -808,6 +812,7 @@ public void replayCreateExternalTableFromEvent(ExternalObjectLog log) { db.writeLock(); try { db.replayCreateTableFromEvent(log.getTableName(), log.getTableId()); + db.setLastUpdateTime(log.getLastUpdateTime()); } finally { db.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 9067c69c477f3a..7ad054bb7f72f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -87,6 +87,8 @@ public abstract class ExternalCatalog private boolean initialized = false; @SerializedName(value = "idToDb") protected Map> idToDb = Maps.newConcurrentMap(); + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; // db name does not contains "default_cluster" protected Map dbNameToId = Maps.newConcurrentMap(); private boolean objectCreated = false; @@ -260,6 +262,9 @@ protected void init() { } dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; + long currentTime = System.currentTimeMillis(); + lastUpdateTime = currentTime; + initCatalogLog.setLastUpdateTime(lastUpdateTime); Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } @@ -282,7 +287,7 @@ public final List getSchema(String dbName, String tblName) { if (db.isPresent()) { Optional table = db.get().getTable(tblName); if (table.isPresent()) { - return table.get().initSchema(); + return table.get().initSchemaAndUpdateTime(); } } // return one column with unsupported type. @@ -406,6 +411,14 @@ private void modifyComment(Map props) { props.remove("comment"); } + public long getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(long lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -440,6 +453,7 @@ public void replayInitCatalog(InitCatalogLog log) { } dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; + lastUpdateTime = log.getLastUpdateTime(); initialized = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java index 04d5b06036b494..4ebb137a607c2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java @@ -56,6 +56,9 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "partitionNames") private List partitionNames; + @SerializedName(value = "lastUpdateTime") + private long lastUpdateTime; + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 73fbeeb78112c8..dd30fbf43c95ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -65,6 +65,9 @@ public enum Type { @SerializedName(value = "type") private Type type; + @SerializedName(value = "lastUpdateTime") + private long lastUpdateTime; + public InitCatalogLog() { refreshCount = 0; createCount = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 14cd4410ec4c1e..3a85fb1edc54c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -68,6 +68,9 @@ public enum Type { @SerializedName(value = "type") private Type type; + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; + public InitDatabaseLog() { refreshCount = 0; createCount = 0; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java index 68b7ae49ff1952..56e33f1b1f9907 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java @@ -26,18 +26,18 @@ public class ShowCatalogStmtTest { @Test public void testNormal() throws UserException, AnalysisException { - final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); + final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); ShowCatalogStmt stmt = new ShowCatalogStmt(); stmt.analyze(analyzer); Assert.assertNull(stmt.getCatalogName()); - Assert.assertEquals(6, stmt.getMetaData().getColumnCount()); + Assert.assertEquals(7, stmt.getMetaData().getColumnCount()); Assert.assertEquals("SHOW CATALOGS", stmt.toSql()); stmt = new ShowCatalogStmt(null, "%hive%"); stmt.analyze(analyzer); Assert.assertNull(stmt.getCatalogName()); Assert.assertNotNull(stmt.getPattern()); - Assert.assertEquals(6, stmt.getMetaData().getColumnCount()); + Assert.assertEquals(7, stmt.getMetaData().getColumnCount()); Assert.assertEquals("SHOW CATALOGS LIKE '%hive%'", stmt.toSql()); stmt = new ShowCatalogStmt("testCatalog", null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java new file mode 100644 index 00000000000000..1a663d7a26a6b7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.catalog.external.TestExternalTable; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class RefreshCatalogTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.catalog.RefreshTableTest$RefreshTableProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + @Test + public void testRefreshCatalog() throws Exception { + CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); + // init is 0 + long l1 = test1.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); + TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); + // getDb() triggered init method + long l2 = test1.getLastUpdateTime(); + Assertions.assertTrue(l2 > l1); + Assertions.assertFalse(table.isObjectCreated()); + table.makeSureInitialized(); + Assertions.assertTrue(table.isObjectCreated()); + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + Assertions.assertTrue(refreshCatalogStmt.isInvalidCache()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + // not triggered init method + long l3 = test1.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); + Assertions.assertTrue(table.isObjectCreated()); + test1.getDbNullable("db1").getTables(); + Assertions.assertFalse(table.isObjectCreated()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); + table.makeSureInitialized(); + Assertions.assertTrue(((ExternalCatalog) test1).isInitialized()); + // table.makeSureInitialized() triggered init method + long l4 = test1.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); + } + + public static class RefreshTableProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("tbl11", Lists.newArrayList( + new Column("a11", PrimitiveType.BIGINT), + new Column("a12", PrimitiveType.STRING), + new Column("a13", PrimitiveType.FLOAT))); + tblSchemaMap1.put("tbl12", Lists.newArrayList( + new Column("b21", PrimitiveType.BIGINT), + new Column("b22", PrimitiveType.STRING), + new Column("b23", PrimitiveType.FLOAT))); + MOCKED_META.put("db1", tblSchemaMap1); + // db2 + Map> tblSchemaMap2 = Maps.newHashMap(); + tblSchemaMap2.put("tbl21", Lists.newArrayList( + new Column("c11", PrimitiveType.BIGINT), + new Column("c12", PrimitiveType.STRING), + new Column("c13", PrimitiveType.FLOAT))); + MOCKED_META.put("db2", tblSchemaMap2); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java new file mode 100644 index 00000000000000..23def21689c70d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshDbStmt; +import org.apache.doris.catalog.external.ExternalDatabase; +import org.apache.doris.catalog.external.TestExternalDatabase; +import org.apache.doris.catalog.external.TestExternalTable; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class RefreshDbTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.catalog.RefreshTableTest$RefreshTableProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + @Test + public void testRefreshDatabase() throws Exception { + CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); + TestExternalDatabase db1 = (TestExternalDatabase) test1.getDbNullable("db1"); + long l1 = db1.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); + TestExternalTable table = db1.getTable("tbl11").get(); + long l2 = db1.getLastUpdateTime(); + Assertions.assertTrue(l2 > l1); + Assertions.assertFalse(table.isObjectCreated()); + table.makeSureInitialized(); + Assertions.assertTrue(table.isObjectCreated()); + RefreshDbStmt refreshDbStmt = new RefreshDbStmt("test1", "db1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); + } catch (Exception e) { + // Do nothing + } + long l3 = db1.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); + Assertions.assertTrue(table.isObjectCreated()); + test1.getDbNullable("db1").getTables(); + Assertions.assertFalse(table.isObjectCreated()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); + } catch (Exception e) { + // Do nothing + } + Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); + table.makeSureInitialized(); + long l4 = db1.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); + Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); + } catch (Exception e) { + // Do nothing + } + } + + public static class RefreshTableProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("tbl11", Lists.newArrayList( + new Column("a11", PrimitiveType.BIGINT), + new Column("a12", PrimitiveType.STRING), + new Column("a13", PrimitiveType.FLOAT))); + tblSchemaMap1.put("tbl12", Lists.newArrayList( + new Column("b21", PrimitiveType.BIGINT), + new Column("b22", PrimitiveType.STRING), + new Column("b23", PrimitiveType.FLOAT))); + MOCKED_META.put("db1", tblSchemaMap1); + // db2 + Map> tblSchemaMap2 = Maps.newHashMap(); + tblSchemaMap2.put("tbl21", Lists.newArrayList( + new Column("c11", PrimitiveType.BIGINT), + new Column("c12", PrimitiveType.STRING), + new Column("c13", PrimitiveType.FLOAT))); + MOCKED_META.put("db2", tblSchemaMap2); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 11e37ce238f05b..305bfa32c43cf2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -19,15 +19,11 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; -import org.apache.doris.analysis.RefreshCatalogStmt; -import org.apache.doris.analysis.RefreshDbStmt; import org.apache.doris.analysis.RefreshTableStmt; import org.apache.doris.analysis.TableName; -import org.apache.doris.catalog.external.ExternalDatabase; import org.apache.doris.catalog.external.TestExternalTable; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.DdlExecutor; @@ -68,77 +64,17 @@ protected void runAfterAll() throws Exception { env.getCatalogMgr().dropCatalog(stmt); } - @Test - public void testRefreshCatalog() throws Exception { - CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); - TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); - Assertions.assertFalse(table.isObjectCreated()); - table.makeSureInitialized(); - Assertions.assertTrue(table.isObjectCreated()); - RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); - Assertions.assertTrue(refreshCatalogStmt.isInvalidCache()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertTrue(table.isObjectCreated()); - test1.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); - table.makeSureInitialized(); - Assertions.assertTrue(((ExternalCatalog) test1).isInitialized()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); - } - - @Test - public void testRefreshDatabase() throws Exception { - CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); - TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); - Assertions.assertFalse(table.isObjectCreated()); - table.makeSureInitialized(); - Assertions.assertTrue(table.isObjectCreated()); - RefreshDbStmt refreshDbStmt = new RefreshDbStmt("test1", "db1", null); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertTrue(table.isObjectCreated()); - test1.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); - table.makeSureInitialized(); - Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); - } catch (Exception e) { - // Do nothing - } - } - @Test public void testRefreshTable() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); Assertions.assertFalse(table.isObjectCreated()); + long l1 = table.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); table.makeSureInitialized(); Assertions.assertTrue(table.isObjectCreated()); + long l2 = table.getLastUpdateTime(); + Assertions.assertTrue(l2 == l1); RefreshTableStmt refreshTableStmt = new RefreshTableStmt(new TableName("test1", "db1", "tbl11")); try { DdlExecutor.execute(Env.getCurrentEnv(), refreshTableStmt); @@ -146,6 +82,12 @@ public void testRefreshTable() throws Exception { // Do nothing } Assertions.assertFalse(table.isObjectCreated()); + long l3 = table.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); + table.getFullSchema(); + // only table.getFullSchema() can change table.lastUpdateTime + long l4 = table.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); } public static class RefreshTableProvider implements TestExternalCatalog.TestCatalogProvider { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java index 07e8a679dc19c7..b3052643d2e463 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java @@ -193,11 +193,13 @@ public void testFetchResultNormal() throws AnalysisException { Assert.assertTrue(result instanceof BaseProcResult); Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota", - "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota"), + "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota", "LastUpdateTime"), result.getColumnNames()); List> rows = Lists.newArrayList(); - rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(), "0", "0.000 ", "1024.000 TB", FeConstants.null_string, "0", "1073741824", "1000")); - rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(), "0", "0.000 ", "1024.000 TB", FeConstants.null_string, "0", "1073741824", "1000")); + rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(), "0", "0.000 ", "1024.000 TB", + FeConstants.null_string, "0", "1073741824", "1000", FeConstants.null_string)); + rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(), "0", "0.000 ", "1024.000 TB", + FeConstants.null_string, "0", "1073741824", "1000", FeConstants.null_string)); Assert.assertEquals(rows, result.getRows()); } @@ -228,8 +230,8 @@ public void testFetchResultInvalid() throws AnalysisException { dir = new DbsProcDir(env, catalog); result = dir.fetchResult(); Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota", - "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota"), - result.getColumnNames()); + "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota", "LastUpdateTime"), + result.getColumnNames()); List> rows = Lists.newArrayList(); Assert.assertEquals(rows, result.getRows()); }