Skip to content

Commit

Permalink
[feat](fe) Show db total size and recycle bin size with ShowDataStmt
Browse files Browse the repository at this point in the history
  • Loading branch information
@SWJTU-ZhangLei
SWJTU-ZhangLei committed Jan 31, 2024
1 parent 2eafef4 commit 644868d
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 5 deletions.
83 changes: 83 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
Expand All @@ -38,6 +39,7 @@
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;

import com.google.common.base.Strings;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
Expand All @@ -54,6 +56,16 @@
import java.util.TreeSet;

public class ShowDataStmt extends ShowStmt {
private static final ShowResultSetMetaData SHOW_DATABASE_DATA_META_DATA =
ShowResultSetMetaData.builder()
.addColumn(new Column("DbId", ScalarType.createVarchar(20)))
.addColumn(new Column("DbName", ScalarType.createVarchar(20)))
.addColumn(new Column("Size", ScalarType.createVarchar(30)))
.addColumn(new Column("RemoteSize", ScalarType.createVarchar(30)))
.addColumn(new Column("RecycleSize", ScalarType.createVarchar(30)))
.addColumn(new Column("RecycleRemoteSize", ScalarType.createVarchar(30)))
.build();

private static final ShowResultSetMetaData SHOW_TABLE_DATA_META_DATA =
ShowResultSetMetaData.builder()
.addColumn(new Column("TableName", ScalarType.createVarchar(20)))
Expand All @@ -71,6 +83,11 @@ public class ShowDataStmt extends ShowStmt {
.addColumn(new Column("RowCount", ScalarType.createVarchar(20)))
.addColumn(new Column("RemoteSize", ScalarType.createVarchar(30)))
.build();

public static final ImmutableList<String> SHOW_DATABASE_DATA_META_DATA_ORIGIN =
new ImmutableList.Builder<String>().add("DbId").add("DbName").add("Size")
.add("RemoteSize").add("RecycleSize").add("RecycleRemoteSize").build();

public static final ImmutableList<String> SHOW_TABLE_DATA_META_DATA_ORIGIN =
new ImmutableList.Builder<String>().add("TableName").add("Size").add("ReplicaCount")
.add("RemoteSize").build();
Expand All @@ -97,6 +114,69 @@ public ShowDataStmt(TableName tableName, List<OrderByElement> orderByElements) {
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
List<String> dbNames = Env.getCurrentInternalCatalog().getDbNames();
if (dbNames == null || dbNames.isEmpty()) {
return;
}
long totalSize = 0;
long totalRemoteSize = 0;
long totalRecycleSize = 0;
long totalRecycleRemoteSize = 0;
Map<Long, Pair<Long, Long>> dbToRecycleSize = Env.getCurrentRecycleBin().getDbToRecycleSize();
// show all database datasize
for (String dbName : dbNames) {
DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbName);
if (db == null) {
continue;
}
List<String> dbInfo = new ArrayList<>();
db.readLock();
try {
dbInfo.add(String.valueOf(db.getId()));
dbInfo.add(dbName);
Pair<Long, Long> usedSize = ((Database) db).getUsedDataSize();
dbInfo.add(String.valueOf(usedSize.first));
dbInfo.add(String.valueOf(usedSize.second));
totalSize += usedSize.first;
totalRemoteSize += usedSize.second;
} finally {
db.readUnlock();
}

Pair<Long, Long> recycleSize = dbToRecycleSize.getOrDefault(db.getId(), Pair.of(0L, 0L));
dbInfo.add(String.valueOf(recycleSize.first));
dbInfo.add(String.valueOf(recycleSize.second));
totalRecycleSize += recycleSize.first;
totalRecycleRemoteSize += recycleSize.second;
dbToRecycleSize.remove(db.getId());
totalRows.add(dbInfo);
}

for (Map.Entry<Long, Pair<Long, Long>> entry : dbToRecycleSize.entrySet()) {
List<String> dbInfo = new ArrayList<>();
dbInfo.add(String.valueOf(entry.getKey()));
dbInfo.add("NULL");
dbInfo.add("0");
dbInfo.add("0");
dbInfo.add(String.valueOf(entry.getValue().first));
dbInfo.add(String.valueOf(entry.getValue().second));
totalRecycleSize += entry.getValue().first;
totalRecycleRemoteSize += entry.getValue().second;
totalRows.add(dbInfo);
}

// calc total size
List<String> dbInfo = new ArrayList<>();
dbInfo.add("Total");
dbInfo.add("NULL");
dbInfo.add(String.valueOf(totalSize));
dbInfo.add(String.valueOf(totalRemoteSize));
dbInfo.add(String.valueOf(totalRecycleSize));
dbInfo.add(String.valueOf(totalRecycleRemoteSize));
totalRows.add(dbInfo);
return;
}
if (tableName != null) {
tableName.analyze(analyzer);
// disallow external catalog
Expand Down Expand Up @@ -374,6 +454,9 @@ public List<List<String>> getResultRows() throws AnalysisException {

@Override
public ShowResultSetMetaData getMetaData() {
if (Strings.isNullOrEmpty(dbName)) {
return SHOW_DATABASE_DATA_META_DATA;
}
if (tableName != null) {
return SHOW_INDEX_DATA_META_DATA;
} else {
Expand Down
41 changes: 41 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DynamicPartitionUtil;
Expand All @@ -48,6 +49,7 @@
import java.io.DataOutput;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -1058,6 +1060,45 @@ public List<List<String>> getInfo() {
return Stream.of(dbInfos, tableInfos, partitionInfos).flatMap(Collection::stream).collect(Collectors.toList());
}

public synchronized Map<Long, Pair<Long, Long>> getDbToRecycleSize() {
Map<Long, Pair<Long, Long>> dbToRecycleSize = new HashMap<>();
for (Map.Entry<Long, RecycleTableInfo> entry : idToTable.entrySet()) {
RecycleTableInfo tableInfo = entry.getValue();
Table table = tableInfo.getTable();
if (!(table instanceof OlapTable)) {
continue;
}
long dataSize = table.getDataSize(false);
long remoteDataSize = ((OlapTable) table).getRemoteDataSize();
dbToRecycleSize.compute(tableInfo.getDbId(), (k, v) -> {
if (v == null) {
return Pair.of(dataSize, remoteDataSize);
} else {
v.first += dataSize;
v.second += remoteDataSize;
return v;
}
});
}

for (Map.Entry<Long, RecyclePartitionInfo> entry : idToPartition.entrySet()) {
RecyclePartitionInfo partitionInfo = entry.getValue();
Partition partition = partitionInfo.getPartition();
long dataSize = partition.getDataSize(false);
long remoteDataSize = partition.getRemoteDataSize();
dbToRecycleSize.compute(partitionInfo.getDbId(), (k, v) -> {
if (v == null) {
return Pair.of(dataSize, remoteDataSize);
} else {
v.first += dataSize;
v.second += remoteDataSize;
return v;
}
});
}
return dbToRecycleSize;
}

// Need to add "synchronized", because when calling /dump api to dump image,
// this class is not protected by any lock, will throw ConcurrentModificationException.
@Override
Expand Down
20 changes: 15 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,10 +272,19 @@ public void setDbProperties(DatabaseProperty dbProperties) {
}

public long getUsedDataQuotaWithLock() {
long usedDataQuota = 0;
return getUsedDataSize().first;
}

public Pair<Long, Long> getUsedDataSize() {
long usedDataSize = 0;
long usedRemoteDataSize = 0;
List<Table> tables = new ArrayList<>();
readLock();
List<Table> tables = new ArrayList<>(this.idToTable.values());
readUnlock();
try {
tables.addAll(this.idToTable.values());
} finally {
readUnlock();
}

for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
Expand All @@ -285,12 +294,13 @@ public long getUsedDataQuotaWithLock() {
OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
usedDataQuota = usedDataQuota + olapTable.getDataSize();
usedDataSize = usedDataSize + olapTable.getDataSize();
usedRemoteDataSize = usedRemoteDataSize + olapTable.getRemoteDataSize();
} finally {
olapTable.readUnlock();
}
}
return usedDataQuota;
return Pair.of(usedDataSize, usedRemoteDataSize);
}

public long getReplicaCountWithLock() {
Expand Down

0 comments on commit 644868d

Please sign in to comment.