From c3d52dba5162ad7119b1442b5b414e569ea362da Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Wed, 4 May 2022 23:21:06 +0200 Subject: [PATCH] make tool compatible with Cassandra 4.1 --- README.md | 2 +- pom.xml | 4 +-- src/deb/control/control | 2 +- .../instaclustr/sstabletools/Histogram.java | 2 +- .../instaclustr/sstabletools/ProgressBar.java | 1 + .../sstabletools/PurgeStatistics.java | 3 +- .../PurgeStatisticsCollector.java | 2 +- .../sstabletools/SSTableMetadata.java | 14 ++++++---- .../sstabletools/SSTableStatistics.java | 21 ++++++++------ .../com/instaclustr/sstabletools/Util.java | 14 ++++++++++ .../cassandra/CassandraBackend.java | 28 +++++++++++-------- .../cassandra/ColumnFamilyBackend.java | 12 ++++---- .../sstabletools/cassandra/DataReader.java | 4 ++- .../sstabletools/cassandra/IndexReader.java | 1 + .../cassandra/PurgeStatisticBackend.java | 17 ++++------- .../cli/SSTableMetadataCollector.java | 2 +- 16 files changed, 79 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 4d4c2c2..1e5efdc 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Commands: cfstats Detailed statistics about cells in a column family pstats Partition size statistics for a column family purge Statistics about reclaimable data for a column family - sstables Print out metadata for sstables the belong to a column family + sstables Print out metadata for sstables that belong to a column family summary Summary information about all column families including how much of the data is repaired ``` diff --git a/pom.xml b/pom.xml index 3bfc010..ac3657d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.instaclustr - ic-sstable-tools-4.0.0 + ic-sstable-tools-4.1.0 1.0.0 Instaclustr SSTable Tools @@ -88,7 +88,7 @@ org.apache.cassandra cassandra-all - 4.0.0 + 4.1.0 provided diff --git a/src/deb/control/control b/src/deb/control/control index 60d8901..b5cfd44 100644 --- a/src/deb/control/control +++ b/src/deb/control/control @@ -3,5 +3,5 @@ Version: [[version]] Section: misc Priority: optional Architecture: all -Depends: cassandra (>= 4.0) +Depends: cassandra (>= 4.1) Maintainer: [[maintainer]] diff --git a/src/main/java/com/instaclustr/sstabletools/Histogram.java b/src/main/java/com/instaclustr/sstabletools/Histogram.java index 20f5ddb..cc98ecc 100644 --- a/src/main/java/com/instaclustr/sstabletools/Histogram.java +++ b/src/main/java/com/instaclustr/sstabletools/Histogram.java @@ -81,7 +81,7 @@ protected int size() { } /** - * Snapshot histogram. + * @return snapshot histogram. */ public Snapshot snapshot() { final int s = size(); diff --git a/src/main/java/com/instaclustr/sstabletools/ProgressBar.java b/src/main/java/com/instaclustr/sstabletools/ProgressBar.java index 992d243..769b889 100644 --- a/src/main/java/com/instaclustr/sstabletools/ProgressBar.java +++ b/src/main/java/com/instaclustr/sstabletools/ProgressBar.java @@ -33,6 +33,7 @@ public final class ProgressBar { * Construct progress bar. * * @param title Progress bar title + * @param interactive whether bar is interactive or not */ public ProgressBar(String title, boolean interactive) { this.title = title; diff --git a/src/main/java/com/instaclustr/sstabletools/PurgeStatistics.java b/src/main/java/com/instaclustr/sstabletools/PurgeStatistics.java index a24ec59..6634870 100644 --- a/src/main/java/com/instaclustr/sstabletools/PurgeStatistics.java +++ b/src/main/java/com/instaclustr/sstabletools/PurgeStatistics.java @@ -1,6 +1,7 @@ package com.instaclustr.sstabletools; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.io.sstable.SSTableId; import java.util.ArrayList; import java.util.Comparator; @@ -26,7 +27,7 @@ public int compare(PurgeStatistics o1, PurgeStatistics o2) { /** * List of generations the key belongs to. */ - public List generations = new ArrayList<>(); + public List ssTableIds = new ArrayList<>(); /** * Size in bytes of current partition. diff --git a/src/main/java/com/instaclustr/sstabletools/PurgeStatisticsCollector.java b/src/main/java/com/instaclustr/sstabletools/PurgeStatisticsCollector.java index 4396f5b..6f37069 100644 --- a/src/main/java/com/instaclustr/sstabletools/PurgeStatisticsCollector.java +++ b/src/main/java/com/instaclustr/sstabletools/PurgeStatisticsCollector.java @@ -94,7 +94,7 @@ public void run() { cfProxy.formatKey(stats.key), Util.humanReadableByteCount(stats.size), Util.humanReadableByteCount(stats.reclaimable), - stats.generations.toString() + stats.ssTableIds.toString() ); } System.out.println(tb); diff --git a/src/main/java/com/instaclustr/sstabletools/SSTableMetadata.java b/src/main/java/com/instaclustr/sstabletools/SSTableMetadata.java index c99b35b..0b25444 100644 --- a/src/main/java/com/instaclustr/sstabletools/SSTableMetadata.java +++ b/src/main/java/com/instaclustr/sstabletools/SSTableMetadata.java @@ -1,7 +1,11 @@ package com.instaclustr.sstabletools; +import org.apache.cassandra.io.sstable.SSTableId; + import java.util.Comparator; +import static com.instaclustr.sstabletools.Util.compareIds; + /** * Metadata statistics about sstable. */ @@ -10,7 +14,7 @@ public class SSTableMetadata { @Override public int compare(SSTableMetadata o1, SSTableMetadata o2) { int cmp = Long.compare(o1.minTimestamp, o2.minTimestamp); - return cmp == 0 ? Integer.compare(o1.generation, o2.generation) : cmp; + return compareIds(cmp, o1.ssTableId, o2.ssTableId); } }; @@ -18,14 +22,14 @@ public int compare(SSTableMetadata o1, SSTableMetadata o2) { @Override public int compare(SSTableMetadata o1, SSTableMetadata o2) { int cmp = Long.compare(o1.maxTimestamp, o2.maxTimestamp); - return cmp == 0 ? Integer.compare(o1.generation, o2.generation) : cmp; + return compareIds(cmp, o1.ssTableId, o2.ssTableId); } }; public final static Comparator GENERATION_COMPARATOR = new Comparator() { @Override public int compare(SSTableMetadata o1, SSTableMetadata o2) { - return Integer.compare(o1.generation, o2.generation); + return compareIds(0, o1.ssTableId, o2.ssTableId); } }; @@ -33,7 +37,7 @@ public int compare(SSTableMetadata o1, SSTableMetadata o2) { @Override public int compare(SSTableMetadata o1, SSTableMetadata o2) { int cmp = Long.compare(o1.level, o2.level); - return cmp == 0 ? Integer.compare(o1.generation, o2.generation) : cmp; + return compareIds(cmp, o1.ssTableId, o2.ssTableId); } }; @@ -45,7 +49,7 @@ public int compare(SSTableMetadata o1, SSTableMetadata o2) { /** * SSTable generation. */ - public int generation; + public SSTableId ssTableId; public long minTimestamp; diff --git a/src/main/java/com/instaclustr/sstabletools/SSTableStatistics.java b/src/main/java/com/instaclustr/sstabletools/SSTableStatistics.java index a3dcbb0..a3df759 100644 --- a/src/main/java/com/instaclustr/sstabletools/SSTableStatistics.java +++ b/src/main/java/com/instaclustr/sstabletools/SSTableStatistics.java @@ -1,16 +1,21 @@ package com.instaclustr.sstabletools; +import org.apache.cassandra.io.sstable.SSTableId; +import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; +import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; + import java.util.Comparator; /** * SSTable statistics. */ public class SSTableStatistics { + public final static Comparator LIVENESS_COMPARATOR = new Comparator() { @Override public int compare(SSTableStatistics o1, SSTableStatistics o2) { int cmp = Long.compare(o1.getLiveness(), o2.getLiveness()); - return cmp == 0 ? Integer.compare(o1.generation, o2.generation) : cmp; + return Util.compareIds(cmp, o1.ssTableId, o2.ssTableId); } }; @@ -18,7 +23,7 @@ public int compare(SSTableStatistics o1, SSTableStatistics o2) { @Override public int compare(SSTableStatistics o1, SSTableStatistics o2) { int cmp = Long.compare(o1.minTimestamp, o2.minTimestamp); - return cmp == 0 ? Integer.compare(o1.generation, o2.generation) : cmp; + return Util.compareIds(cmp, o1.ssTableId, o2.ssTableId); } }; @@ -26,14 +31,14 @@ public int compare(SSTableStatistics o1, SSTableStatistics o2) { @Override public int compare(SSTableStatistics o1, SSTableStatistics o2) { int cmp = Long.compare(o1.maxTimestamp, o2.maxTimestamp); - return cmp == 0 ? Integer.compare(o1.generation, o2.generation) : cmp; + return Util.compareIds(cmp, o1.ssTableId, o2.ssTableId); } }; /** - * SSTable generation. + * SSTable id. */ - public int generation; + public SSTableId ssTableId; /** * File name of SSTable Data.db. @@ -123,15 +128,15 @@ public int compare(SSTableStatistics o1, SSTableStatistics o2) { /** * Construct statistics record for SSTable. * - * @param generation SSTable Generation + * @param ssTableId SSTable id * @param filename Filename of Data.db * @param uncompressedLength Uncompressed length of SSTable Data.db in bytes * @param minTimestamp Minimum timestamp of SSTable * @param maxTimestamp Maximum timestamp of SSTable * @param level SSTable LTCS level */ - public SSTableStatistics(int generation, String filename, long uncompressedLength, long minTimestamp, long maxTimestamp, int level) { - this.generation = generation; + public SSTableStatistics(SSTableId ssTableId, String filename, long uncompressedLength, long minTimestamp, long maxTimestamp, int level) { + this.ssTableId = ssTableId; this.filename = filename; this.size = uncompressedLength; this.minTimestamp = minTimestamp; diff --git a/src/main/java/com/instaclustr/sstabletools/Util.java b/src/main/java/com/instaclustr/sstabletools/Util.java index e7fff36..2ce763f 100644 --- a/src/main/java/com/instaclustr/sstabletools/Util.java +++ b/src/main/java/com/instaclustr/sstabletools/Util.java @@ -1,5 +1,9 @@ package com.instaclustr.sstabletools; +import org.apache.cassandra.io.sstable.SSTableId; +import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; +import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; + import java.text.SimpleDateFormat; import java.util.Random; import java.util.TimeZone; @@ -25,6 +29,16 @@ public final class Util { rand = new Random(); } + public static int compareIds(int cmp, SSTableId o1, SSTableId o2) { + if (o1 instanceof UUIDBasedSSTableId) { + return cmp == 0 ? ((UUIDBasedSSTableId) o1).compareTo((UUIDBasedSSTableId) o2) : cmp; + } else if (o1 instanceof SequenceBasedSSTableId) { + return cmp == 0 ? ((SequenceBasedSSTableId) o1).compareTo((SequenceBasedSSTableId) o2) : cmp; + } else { + throw new IllegalStateException("Unable to process SSTableId of type " + o1.getClass()); + } + } + public static String humanReadableByteCount(long bytes) { return humanReadableByteCount(bytes, true); } diff --git a/src/main/java/com/instaclustr/sstabletools/cassandra/CassandraBackend.java b/src/main/java/com/instaclustr/sstabletools/cassandra/CassandraBackend.java index 9f81c3a..a3da7e1 100644 --- a/src/main/java/com/instaclustr/sstabletools/cassandra/CassandraBackend.java +++ b/src/main/java/com/instaclustr/sstabletools/cassandra/CassandraBackend.java @@ -21,9 +21,10 @@ import java.io.IOException; import java.nio.file.Files; import java.util.*; +import java.util.stream.Collectors; /** - * Proxy to Cassandra 3.0 backend. + * Proxy to Cassandra 4.1 backend. */ public class CassandraBackend implements CassandraProxy { private static final CassandraBackend singleton = new CassandraBackend(); @@ -34,19 +35,22 @@ public static CassandraProxy getInstance() { static { Util.initDatabaseDescriptor(); - Schema.instance.loadFromDisk(false); + Schema.instance.loadFromDisk(); } private CassandraBackend() {} public List getKeyspaces() { - List names = new ArrayList<>(Schema.instance.getNonSystemKeyspaces()); - Collections.sort(names); - return names; + return Schema.instance.getNonLocalStrategyKeyspaces() + .stream() + .map(ksmd -> ksmd.name).sorted().collect(Collectors.toList()); } public List getColumnFamilies(String ksName) { KeyspaceMetadata ksMetaData = Schema.instance.getKeyspaceMetadata(ksName); + if (ksMetaData == null) { + throw new IllegalStateException("Unknown keyspace " + ksMetaData.name); + } List names = new ArrayList<>(ksMetaData.tables.size() + ksMetaData.views.size()); for (TableMetadata cfMetaData : ksMetaData.tablesAndViews()) { names.add(cfMetaData.name); @@ -58,7 +62,7 @@ public List getColumnFamilies(String ksName) { private ColumnFamilyStore getStore(String ksName, String cfName) { // Start by validating keyspace name if (Schema.instance.getKeyspaceMetadata(ksName) == null) { - System.err.println(String.format("Reference to nonexistent keyspace: %s!", ksName)); + System.err.printf("Reference to nonexistent keyspace: %s!%n", ksName); System.exit(1); } Keyspace keyspace = Keyspace.open(ksName); @@ -74,9 +78,9 @@ private ColumnFamilyStore getStore(String ksName, String cfName) { try { return keyspace.getColumnFamilyStore(baseName); } catch (Throwable t) { - System.err.println(String.format( - "The provided column family is not part of this cassandra keyspace: keyspace = %s, column family = %s", - ksName, cfName)); + System.err.printf( + "The provided column family is not part of this cassandra keyspace: keyspace = %s, column family = %s%n", + ksName, cfName); System.exit(1); } return null; @@ -90,7 +94,7 @@ public List getSSTableMetadata(String ksName, String cfName) { SSTableMetadata tableMetadata = new SSTableMetadata(); File dataFile = new File(table.descriptor.filenameFor(Component.DATA)); tableMetadata.filename = dataFile.getName(); - tableMetadata.generation = table.descriptor.generation; + tableMetadata.ssTableId = table.descriptor.id; try { tableMetadata.fileTimestamp = Files.getLastModifiedTime(dataFile.toPath()).toMillis(); } catch (IOException e) { @@ -131,7 +135,7 @@ public ColumnFamilyProxy getColumnFamily(String ksName, String cfName, String sn snapshotName, filter); } catch (Throwable t) { - System.err.println(String.format("Error retrieving snapshot for %s.%s", ksName, cfName)); + System.err.printf("Error retrieving snapshot for %s.%s%n", ksName, cfName); System.exit(1); } @@ -144,7 +148,7 @@ public Class getCompactionClass(String ksName, String cfName) { TableMetadata metaData = Schema.instance.getTableMetadata(ksName, cfName); return metaData.params.compaction.klass(); } catch (Throwable t) { - System.err.println(String.format("Error retrieving snapshot for %s.%s", ksName, cfName)); + System.err.printf("Error retrieving snapshot for %s.%s%n", ksName, cfName); System.exit(1); } diff --git a/src/main/java/com/instaclustr/sstabletools/cassandra/ColumnFamilyBackend.java b/src/main/java/com/instaclustr/sstabletools/cassandra/ColumnFamilyBackend.java index cc30d84..5c0729a 100644 --- a/src/main/java/com/instaclustr/sstabletools/cassandra/ColumnFamilyBackend.java +++ b/src/main/java/com/instaclustr/sstabletools/cassandra/ColumnFamilyBackend.java @@ -1,6 +1,5 @@ package com.instaclustr.sstabletools.cassandra; -import com.google.common.util.concurrent.RateLimiter; import com.instaclustr.sstabletools.*; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; @@ -9,6 +8,7 @@ import java.io.File; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -66,7 +66,7 @@ public ColumnFamilyBackend(AbstractType keyValidator, this.clearSnapshot = false; } else { snapshotName = Util.generateSnapshotName(); - cfStore.snapshotWithoutFlush(snapshotName, null, true, null); + cfStore.snapshotWithoutMemtable(snapshotName, null, true, null, null, Instant.now()); this.clearSnapshot = true; } this.snapshotName = snapshotName; @@ -91,7 +91,7 @@ public Collection getIndexReaders() { File dataFile = new File(sstable.descriptor.filenameFor(Component.DATA)); readers.add(new IndexReader( new SSTableStatistics( - sstable.descriptor.generation, + sstable.descriptor.id, dataFile.getName(), sstable.uncompressedLength(), sstable.getMinTimestamp(), @@ -101,7 +101,9 @@ public Collection getIndexReaders() { sstable.descriptor.version, sstable.getPartitioner() )); - } catch (Throwable t) {} + } catch (Throwable t) { + + } } return readers; } @@ -114,7 +116,7 @@ public Collection getDataReaders() { File dataFile = new File(sstable.descriptor.filenameFor(Component.DATA)); readers.add(new DataReader( new SSTableStatistics( - sstable.descriptor.generation, + sstable.descriptor.id, dataFile.getName(), sstable.uncompressedLength(), sstable.getMinTimestamp(), diff --git a/src/main/java/com/instaclustr/sstabletools/cassandra/DataReader.java b/src/main/java/com/instaclustr/sstabletools/cassandra/DataReader.java index ae3af58..0775953 100644 --- a/src/main/java/com/instaclustr/sstabletools/cassandra/DataReader.java +++ b/src/main/java/com/instaclustr/sstabletools/cassandra/DataReader.java @@ -29,9 +29,11 @@ public class DataReader extends AbstractSSTableReader { private long position; /** - * Construct a reader for Index.db sstable file. + * Construct a reader for Data.db sstable file. * * @param tableStats SSTable statistics. + * @param scanner scanner of sstables + * @param gcGrace gc_grace of table */ public DataReader(SSTableStatistics tableStats, ISSTableScanner scanner, int gcGrace) { this.tableStats = tableStats; diff --git a/src/main/java/com/instaclustr/sstabletools/cassandra/IndexReader.java b/src/main/java/com/instaclustr/sstabletools/cassandra/IndexReader.java index 446be14..b88c502 100644 --- a/src/main/java/com/instaclustr/sstabletools/cassandra/IndexReader.java +++ b/src/main/java/com/instaclustr/sstabletools/cassandra/IndexReader.java @@ -50,6 +50,7 @@ public class IndexReader extends AbstractSSTableReader { * * @param tableStats SSTable statistics. * @param reader Reader to Index.db file. + * @param version Version of SSTable * @param partitioner The sstable partitioner. */ public IndexReader(SSTableStatistics tableStats, RandomAccessReader reader, Version version, IPartitioner partitioner) { diff --git a/src/main/java/com/instaclustr/sstabletools/cassandra/PurgeStatisticBackend.java b/src/main/java/com/instaclustr/sstabletools/cassandra/PurgeStatisticBackend.java index 0322493..56b3956 100644 --- a/src/main/java/com/instaclustr/sstabletools/cassandra/PurgeStatisticBackend.java +++ b/src/main/java/com/instaclustr/sstabletools/cassandra/PurgeStatisticBackend.java @@ -1,19 +1,14 @@ package com.instaclustr.sstabletools.cassandra; -import com.google.common.util.concurrent.RateLimiter; import com.instaclustr.sstabletools.PurgeStatistics; import com.instaclustr.sstabletools.PurgeStatisticsReader; import com.instaclustr.sstabletools.Util; -import org.apache.cassandra.db.rows.UnfilteredRowIterators.MergeListener; -import org.apache.cassandra.io.sstable.format.big.BigTableReader; -import org.apache.cassandra.io.sstable.format.big.BigTableScanner; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.*; import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.partitions.PurgeFunction; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; @@ -54,7 +49,7 @@ public PurgeStatisticBackend(ColumnFamilyStore cfs, Collection(sstables.size()); for (org.apache.cassandra.io.sstable.format.SSTableReader sstable : sstables) { length += sstable.uncompressedLength(); - ScannerWrapper scanner = new ScannerWrapper(sstable.descriptor.generation, sstable.getScanner()); + ScannerWrapper scanner = new ScannerWrapper(sstable.descriptor.id, sstable.getScanner()); if (scanner.next()) { readerQueue.add(scanner); } @@ -127,7 +122,7 @@ private void onUnfiltered(Unfiltered unfiltered) { } }); rows.add(row); - stats.generations.add(scannerWrapper.generation); + stats.ssTableIds.add(scannerWrapper.ssTableId); } // Merge rows together and grab column statistics. @@ -285,7 +280,7 @@ private class ScannerWrapper implements Comparable { /** * Generation of sstable being scanned. */ - public int generation; + public SSTableId ssTableId; /** * SSTable scanner. @@ -302,8 +297,8 @@ private class ScannerWrapper implements Comparable { */ private long position; - public ScannerWrapper(int generation, ISSTableScanner scanner) { - this.generation = generation; + public ScannerWrapper(SSTableId ssTableId, ISSTableScanner scanner) { + this.ssTableId = ssTableId; this.scanner = scanner; this.position = 0; } diff --git a/src/main/java/com/instaclustr/sstabletools/cli/SSTableMetadataCollector.java b/src/main/java/com/instaclustr/sstabletools/cli/SSTableMetadataCollector.java index 63f3dfd..4b900f0 100644 --- a/src/main/java/com/instaclustr/sstabletools/cli/SSTableMetadataCollector.java +++ b/src/main/java/com/instaclustr/sstabletools/cli/SSTableMetadataCollector.java @@ -23,7 +23,7 @@ versionProvider = CLI.class, name = "sstables", usageHelpWidth = 128, - description = "Print out metadata for sstables the belong to a column family", + description = "Print out metadata for sstables that belong to a column family", mixinStandardHelpOptions = true ) public class SSTableMetadataCollector implements Runnable {