update the tool to support Cassandra 5, updated picocli and fix versi…

…on parsing

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>com.instaclustr</groupId>
-    <artifactId>ic-sstable-tools-4.1.3</artifactId>
+    <artifactId>ic-sstable-tools-5.0.0</artifactId>
     <version>1.0.0</version>
 
     <name>Instaclustr SSTable Tools</name>
@@ -21,7 +21,7 @@
         <maven.jar.plugin.version>3.1.1</maven.jar.plugin.version>
         <maven.javadoc.plugin.version>3.1.1</maven.javadoc.plugin.version>
         <maven.compiler.plugin.version>3.8.1</maven.compiler.plugin.version>
-        <maven.git.command.plugin.version>2.2.4</maven.git.command.plugin.version>
+        <maven.git.command.plugin.version>4.9.10</maven.git.command.plugin.version>
 
         <maintainer>Various &lt;support@instaclustr.com&gt;</maintainer>
         <version.jdeb>1.8</version.jdeb>
@@ -34,9 +34,9 @@
         <outputDirectory>${project.build.directory}</outputDirectory>
 
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
-        <java.version>8</java.version>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <java.version>11</java.version>
     </properties>
 
     <inceptionYear>2018</inceptionYear>
@@ -83,12 +83,12 @@
         <dependency>
             <groupId>info.picocli</groupId>
             <artifactId>picocli</artifactId>
-            <version>4.5.2</version>
+            <version>4.7.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.cassandra</groupId>
             <artifactId>cassandra-all</artifactId>
-            <version>4.1.3</version>
+            <version>5.0-beta1</version>
             <scope>provided</scope>
         </dependency>
     </dependencies>
@@ -109,7 +109,7 @@
                     </execution>
                 </executions>
                 <configuration>
-                    <source>8</source>
+                    <source>${java.version}</source>
                 </configuration>
             </plugin>
             <plugin>
@@ -147,13 +147,21 @@
                 <version>${maven.git.command.plugin.version}</version>
                 <executions>
                     <execution>
+                        <id>get-the-git-infos</id>
                         <goals>
                             <goal>revision</goal>
                         </goals>
+                        <phase>initialize</phase>
                     </execution>
                 </executions>
                 <configuration>
-                    <dotGitDirectory>${project.basedir}/.git</dotGitDirectory>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
+                    <includeOnlyProperties>
+                        <includeOnlyProperty>^git.build.(time|version)$</includeOnlyProperty>
+                        <includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
+                    </includeOnlyProperties>
+                    <commitIdGenerationMode>full</commitIdGenerationMode>
                 </configuration>
             </plugin>
             <plugin>

src/main/java/com/instaclustr/sstabletools/ColumnFamilyProxy.java

@@ -1,6 +1,5 @@
 package com.instaclustr.sstabletools;
 
-import com.google.common.util.concurrent.RateLimiter;
 import org.apache.cassandra.db.DecoratedKey;
 
 import java.util.Collection;
@@ -39,13 +38,6 @@ public interface ColumnFamilyProxy extends AutoCloseable {
      */
     String formatKey(DecoratedKey key);
 
-    /**
-     * Is the column family using Date Tiered compaction strategy.
-     *
-     * @return True if column family is using Date Tiered compaction strategy.
-     */
-    boolean isDTCS();
-
     /**
      * Is the column family using Time Window compaction strategy.
      *

src/main/java/com/instaclustr/sstabletools/PurgeStatisticsCollector.java

@@ -29,7 +29,7 @@ public class PurgeStatisticsCollector implements Runnable {
     @Option(names = {"-t"}, description = "Snapshot name", arity = "1")
     public String snapshotName;
 
-    @Option(names = {"-f"}, description = "Filter to sstables (comma separated", defaultValue = "")
+    @Option(names = {"-f"}, description = "Filter to sstables (comma separated)", defaultValue = "")
     public String filters;
 
     @Option(names = {"-b"}, description = "Batch mode", arity = "0")

src/main/java/com/instaclustr/sstabletools/SSTableMetadata.java

@@ -55,9 +55,9 @@ public int compare(SSTableMetadata o1, SSTableMetadata o2) {
 
     public long maxTimestamp;
 
-    public int minLocalDeletionTime;
+    public long minLocalDeletionTime;
 
-    public int maxLocalDeletionTime;
+    public long maxLocalDeletionTime;
 
     public long fileTimestamp;
 

src/main/java/com/instaclustr/sstabletools/cassandra/CassandraBackend.java

@@ -3,11 +3,11 @@
 import com.instaclustr.sstabletools.CassandraProxy;
 import com.instaclustr.sstabletools.ColumnFamilyProxy;
 import com.instaclustr.sstabletools.SSTableMetadata;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.compaction.DateTieredCompactionStrategy;
 import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -41,7 +41,7 @@ public static CassandraProxy getInstance() {
     private CassandraBackend() {}
 
     public List<String> getKeyspaces() {
-        return Schema.instance.getNonLocalStrategyKeyspaces()
+        return Schema.instance.distributedKeyspaces()
                      .stream()
                      .map(ksmd -> ksmd.name).sorted().collect(Collectors.toList());
     }
@@ -92,7 +92,7 @@ public List<SSTableMetadata> getSSTableMetadata(String ksName, String cfName) {
         List<SSTableMetadata> metaData = new ArrayList<>(tables.size());
         for (SSTableReader table : tables) {
             SSTableMetadata tableMetadata = new SSTableMetadata();
-            File dataFile = new File(table.descriptor.filenameFor(Component.DATA));
+            File dataFile = table.descriptor.fileFor(SSTableFormat.Components.DATA).toJavaIOFile();
             tableMetadata.filename = dataFile.getName();
             tableMetadata.ssTableId = table.descriptor.id;
             try {
@@ -129,7 +129,6 @@ public ColumnFamilyProxy getColumnFamily(String ksName, String cfName, String sn
             Class<?> compactionClass = metaData.params.compaction.klass();
             return new ColumnFamilyBackend(
                     metaData.partitionKeyType,
-                    compactionClass.equals(DateTieredCompactionStrategy.class),
                     compactionClass.equals(TimeWindowCompactionStrategy.class),
                     cfStore,
                     snapshotName,

src/main/java/com/instaclustr/sstabletools/cassandra/ColumnFamilyBackend.java

@@ -3,30 +3,38 @@
 import com.instaclustr.sstabletools.*;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.FilterFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Optional;
+import java.util.Set;
 
 /**
  * ColumnFamilyProxy using Cassandra 3.5 backend.
  */
 public class ColumnFamilyBackend implements ColumnFamilyProxy {
+
+    private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyBackend.class);
+
     /**
      * Key validator for column family.
      */
     private AbstractType<?> keyValidator;
 
-    /**
-     * Is column family using Date Tiered Compaction Strategy.
-     */
-    private boolean isDTCS;
-
     /**
      * Is column family using Time Window Compaction Strategy.
      */
@@ -53,13 +61,11 @@ public class ColumnFamilyBackend implements ColumnFamilyProxy {
     private Collection<org.apache.cassandra.io.sstable.format.SSTableReader> sstables;
 
     public ColumnFamilyBackend(AbstractType<?> keyValidator,
-                               boolean isDTCS,
                                boolean isTWCS,
                                ColumnFamilyStore cfStore,
                                String snapshotName,
                                Collection<String> filter) throws IOException {
         this.keyValidator = keyValidator;
-        this.isDTCS = isDTCS;
         this.isTWCS = isTWCS;
         this.cfStore = cfStore;
         if (snapshotName != null) {
@@ -74,7 +80,7 @@ public ColumnFamilyBackend(AbstractType<?> keyValidator,
         if (filter != null) {
             List<org.apache.cassandra.io.sstable.format.SSTableReader> filteredSSTables = new ArrayList<>(sstables.size());
             for (org.apache.cassandra.io.sstable.format.SSTableReader sstable : sstables) {
-                File dataFile = new File(sstable.descriptor.filenameFor(Component.DATA));
+                File dataFile = sstable.descriptor.fileFor(SSTableFormat.Components.DATA).toJavaIOFile();;
                 if (filter.contains(dataFile.getName())) {
                     filteredSSTables.add(sstable);
                 }
@@ -88,7 +94,24 @@ public Collection<SSTableReader> getIndexReaders() {
         Collection<SSTableReader> readers = new ArrayList<>(sstables.size());
         for (org.apache.cassandra.io.sstable.format.SSTableReader sstable : sstables) {
             try {
-                File dataFile = new File(sstable.descriptor.filenameFor(Component.DATA));
+                Set<Component> components = sstable.descriptor.discoverComponents();
+
+                Optional<Component> maybeIndexComponent = components.stream().filter(c -> c.name.contains("Index")).findFirst();
+                if (!maybeIndexComponent.isPresent()) {
+                    continue;
+                }
+
+                org.apache.cassandra.io.util.File indexFile = sstable.descriptor.fileFor(maybeIndexComponent.get());
+                FileHandle indexHandle = new FileHandle.Builder(indexFile).complete();
+
+                BigTableReader reader = new BigTableReader.Builder(sstable.descriptor)
+                        .setComponents(components)
+                        .setFilter(FilterFactory.AlwaysPresent)
+                        .setSerializationHeader(SerializationHeader.makeWithoutStats(cfStore.metadata()))
+                        .setIndexFile(indexHandle)
+                        .build(this.cfStore, false, false);
+
+                File dataFile = sstable.descriptor.fileFor(SSTableFormat.Components.DATA).toJavaIOFile();
                 readers.add(new IndexReader(
                         new SSTableStatistics(
                                 sstable.descriptor.id,
@@ -97,12 +120,12 @@ public Collection<SSTableReader> getIndexReaders() {
                                 sstable.getMinTimestamp(),
                                 sstable.getMaxTimestamp(),
                                 sstable.getSSTableLevel()),
-                        sstable.openIndexReader(),
+                        reader.getIndexFile().createReader(),
                         sstable.descriptor.version,
                         sstable.getPartitioner()
                 ));
             } catch (Throwable t) {
-
+                logger.error("Error opening index readers", t);
             }
         }
         return readers;
@@ -113,7 +136,7 @@ public Collection<SSTableReader> getDataReaders() {
         Collection<SSTableReader> readers = new ArrayList<>(sstables.size());
         for (org.apache.cassandra.io.sstable.format.SSTableReader sstable : sstables) {
             try {
-                File dataFile = new File(sstable.descriptor.filenameFor(Component.DATA));
+                File dataFile = sstable.descriptor.fileFor(SSTableFormat.Components.DATA).toJavaIOFile();
                 readers.add(new DataReader(
                         new SSTableStatistics(
                                 sstable.descriptor.id,
@@ -125,7 +148,9 @@ public Collection<SSTableReader> getDataReaders() {
                         sstable.getScanner(),
                         Util.NOW_SECONDS - sstable.metadata().params.gcGraceSeconds
                 ));
-            } catch (Throwable t) {}
+            } catch (Throwable t) {
+                logger.error("Error while getting data readers", t);
+            }
         }
         return readers;
     }
@@ -140,11 +165,6 @@ public String formatKey(DecoratedKey key) {
         return keyValidator.getString(key.getKey());
     }
 
-    @Override
-    public boolean isDTCS() {
-        return isDTCS;
-    }
-
     @Override
     public boolean isTWCS() {
         return isTWCS;

src/main/java/com/instaclustr/sstabletools/cassandra/IndexReader.java

@@ -67,7 +67,7 @@ public IndexReader(SSTableStatistics tableStats, RandomAccessReader reader, Vers
      * @throws IOException
      */
     private void skipData() throws IOException {
-        int size = version.getVersion().compareTo("ma") >= 0 ? (int) reader.readUnsignedVInt() : reader.readInt();
+        int size = version.version.compareTo("ma") >= 0 ? (int) reader.readUnsignedVInt() : reader.readInt();
         if (size > 0) {
             reader.skipBytesFully(size);
         }
@@ -81,14 +81,14 @@ public boolean next() {
         try {
             if (nextKey == null) {
                 nextKey = ByteBufferUtil.readWithShortLength(reader);
-                nextPosition = version.getVersion().compareTo("ma") > 0 ? reader.readUnsignedVInt() : reader.readLong();
+                nextPosition = version.version.compareTo("ma") > 0 ? reader.readUnsignedVInt() : reader.readLong();
                 skipData();
             }
             partitionStats = new PartitionStatistics(partitioner.decorateKey(nextKey));
             long position = nextPosition;
             if (!reader.isEOF()) {
                 nextKey = ByteBufferUtil.readWithShortLength(reader);
-                nextPosition = version.getVersion().compareTo("ma") > 0 ? reader.readUnsignedVInt() : reader.readLong();
+                nextPosition = version.version.compareTo("ma") > 0 ? reader.readUnsignedVInt() : reader.readLong();
                 skipData();
                 partitionStats.size = nextPosition - position;
             } else {