Percentile filter

common/pom.xml

@@ -14,7 +14,7 @@
  <name>Cassandra Exporter Common</name>
 
  <properties>
- <version.picocli>3.6.1</version.picocli>
+ <version.picocli>3.9.5</version.picocli>
  </properties>
 
  <dependencies>

common/src/main/java/com/zegelin/cassandra/exporter/Harvester.java

@@ -7,6 +7,7 @@
 import com.zegelin.jmx.NamedObject;
 import com.zegelin.cassandra.exporter.cli.HarvesterOptions;
 import com.zegelin.prometheus.domain.CounterMetricFamily;
+import com.zegelin.prometheus.domain.Interval.Quantile;
 import com.zegelin.prometheus.domain.Labels;
 import com.zegelin.prometheus.domain.MetricFamily;
 import com.zegelin.prometheus.domain.NumericMetric;
@@ -132,6 +133,7 @@ public boolean equals(final Object o) {
 
  private final boolean collectorTimingEnabled;
  private final Map<String, Stopwatch> collectionTimes = new ConcurrentHashMap<>();
+ private final Set<Quantile> excludedHistoQuantiles;
 
  private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
  .setNameFormat("cassandra-exporter-harvester-defer-%d")
@@ -145,6 +147,11 @@ protected Harvester(final MetadataFactory metadataFactory, final HarvesterOption
  this.exclusions = options.exclusions;
  this.enabledGlobalLabels = options.globalLabels;
  this.collectorTimingEnabled = options.collectorTimingEnabled;
+ this.excludedHistoQuantiles = options.excludedHistoQuantiles;
+ }
+
+ public Set<Quantile> getExcludedHistoQuantiles() {
+ return excludedHistoQuantiles;
  }
 
  protected void addCollectorFactory(final MBeanGroupMetricFamilyCollector.Factory factory) {

common/src/main/java/com/zegelin/cassandra/exporter/cli/HarvesterOptions.java

@@ -2,6 +2,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.zegelin.netty.Floats;
+import com.zegelin.prometheus.domain.Interval.Quantile;
 import com.zegelin.cassandra.exporter.FactoriesSupplier;
 import com.zegelin.cassandra.exporter.Harvester;
 import picocli.CommandLine;
@@ -152,4 +153,27 @@ public void setExcludeSystemTables(final boolean excludeSystemTables) {
 
  excludedKeyspaces.addAll(CASSANDRA_SYSTEM_KEYSPACES);
  }
+
+ public final Set<Quantile> excludedHistoQuantiles = new HashSet<>();
+ @Option(names = {"--exclude-from-histogram"}, paramLabel = "EXCLUSION", arity = "1..*",
+ description = "Select which quantiles to exclude from histogram metrics. The specified quantiles are excluded from all histogram/summary metrics" +
+ "Valid options are: P_50, P_75, P_95, P_98, P_99, P_99_9" +
+ "'P_50' (Quantile .5), " +
+ "'P_75' (Quantile .75), " +
+ "'P_95' (Quantile .95), " +
+ "'P_98' (Quantile .98). " +
+ "'P_99' (Quantile .99). " +
+ "'P_99_9' (Quantile .999). " +
+ "The default is to include all quantiles. "
+ )
+ void setExcludeFromHistogram(final Set<String> values) {
+ values.forEach( e -> {
+ Quantile q = Quantile.ALL_PERCENTILES.get(e);
+ if(q == null) {
+ throw new IllegalArgumentException(String.format("The specified exlusion quantile '%s' is invalid, value values are '%s'", e, Quantile.ALL_PERCENTILES.keySet()));
+ }
+ excludedHistoQuantiles.add(q); 
+ });
+ }
+
 }

common/src/main/java/com/zegelin/cassandra/exporter/netty/HttpHandler.java

@@ -299,7 +299,7 @@ private ChannelFuture sendMetrics(final ChannelHandlerContext ctx, final FullHtt
  lastWriteFuture = ctx.writeAndFlush(response);
 
  if (request.getMethod() == HttpMethod.GET) {
- ReadableByteChannel byteChannel = new FormattedByteChannel(new TextFormatExposition(metricFamilyStream, timestamp, globalLabels, includeHelp));
+ ReadableByteChannel byteChannel = new FormattedByteChannel(new TextFormatExposition(metricFamilyStream, timestamp, globalLabels, includeHelp, harvester.getExcludedHistoQuantiles()));
  lastWriteFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedNioStream(byteChannel, FormattedByteChannel.MAX_CHUNK_SIZE)));
  }
 

common/src/main/java/com/zegelin/prometheus/domain/Interval.java

@@ -1,13 +1,13 @@
 package com.zegelin.prometheus.domain;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.zegelin.function.FloatFloatFunction;
 
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
 
 /*
 A Summary quanitle or Histogram bucket and associated value.
@@ -23,6 +23,14 @@ public static class Quantile {
 
  public static final Set<Quantile> STANDARD_PERCENTILES = ImmutableSet.of(P_50, P_75, P_95, P_98, P_99, P_99_9);
  public static final Quantile POSITIVE_INFINITY = q(Float.POSITIVE_INFINITY);
+ public static final Map<String,Quantile> ALL_PERCENTILES = new ImmutableMap.Builder<String,Quantile>()
+ .put("P_50",P_50)
+ .put("P_75",P_75)
+ .put("P_95",P_95)
+ .put("P_98",P_98)
+ .put("P_99",P_99)
+ .put("P_99_9",P_99_9)
+ .build();
 
  public final float value;
 

common/src/main/java/com/zegelin/prometheus/exposition/text/TextFormatExposition.java

@@ -3,12 +3,14 @@
 import com.google.common.base.Stopwatch;
 import com.zegelin.netty.Resources;
 import com.zegelin.prometheus.domain.*;
+import com.zegelin.prometheus.domain.Interval.Quantile;
 import com.zegelin.prometheus.exposition.ExpositionSink;
 import com.zegelin.prometheus.exposition.FormattedExposition;
 import io.netty.buffer.ByteBuf;
 
 import java.time.Instant;
 import java.util.Iterator;
+import java.util.Set;
 import java.util.stream.Stream;
 
 public class TextFormatExposition implements FormattedExposition {
@@ -35,13 +37,14 @@ private enum State {
  private int metricCount = 0;
 
  private final Stopwatch stopwatch = Stopwatch.createUnstarted();
-
-
- public TextFormatExposition(final Stream<MetricFamily> metricFamilies, final Instant timestamp, final Labels globalLabels, final boolean includeHelp) {
+ private final Set<Quantile> excludedHistoQuantiles;
+ 
+ public TextFormatExposition(final Stream<MetricFamily> metricFamilies, final Instant timestamp, final Labels globalLabels, final boolean includeHelp, final Set<Quantile> excludedHistoQuantiles) {
  this.metricFamiliesIterator = metricFamilies.iterator();
  this.timestamp = timestamp;
  this.globalLabels = globalLabels;
  this.includeHelp = includeHelp;
+ this.excludedHistoQuantiles = excludedHistoQuantiles;
  }
 
  @Override
@@ -70,7 +73,7 @@ public void nextSlice(final ExpositionSink<?> chunkBuffer) {
 
  final MetricFamily<?> metricFamily = metricFamiliesIterator.next();
 
- metricFamilyWriter = new TextFormatMetricFamilyWriter(timestamp, globalLabels, includeHelp, metricFamily);
+ metricFamilyWriter = new TextFormatMetricFamilyWriter(timestamp, globalLabels, includeHelp, metricFamily, excludedHistoQuantiles);
 
  metricFamilyWriter.writeFamilyHeader(chunkBuffer);
 

common/src/main/java/com/zegelin/prometheus/exposition/text/TextFormatMetricFamilyWriter.java

@@ -3,10 +3,12 @@
 import com.google.common.escape.CharEscaperBuilder;
 import com.google.common.escape.Escaper;
 import com.zegelin.prometheus.domain.*;
+import com.zegelin.prometheus.domain.Interval.Quantile;
 import com.zegelin.prometheus.exposition.ExpositionSink;
 
 import java.time.Instant;
 import java.util.Iterator;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -41,14 +43,16 @@ void write(final ExpositionSink<?> buffer) {
 
  private final Consumer<ExpositionSink<?>> headerWriter;
  private final Function<ExpositionSink<?>, Boolean> metricWriter;
+ private final Set<Quantile> excludedHistoQuantiles;
 
- TextFormatMetricFamilyWriter(final Instant timestamp, final Labels globalLabels, final boolean includeHelp, final MetricFamily<?> metricFamily) {
+ TextFormatMetricFamilyWriter(final Instant timestamp, final Labels globalLabels, final boolean includeHelp, final MetricFamily<?> metricFamily, final Set<Quantile> excludedHistoQuantiles) {
  this.timestamp = " " + timestamp.toEpochMilli();
  this.globalLabels = globalLabels;
  this.includeHelp = includeHelp;
 
  this.headerWriter = metricFamily.accept(new HeaderVisitor());
  this.metricWriter = metricFamily.accept(new MetricVisitor());
+ this.excludedHistoQuantiles=excludedHistoQuantiles;
  }
 
  class HeaderVisitor implements MetricFamilyVisitor<Consumer<ExpositionSink<?>>> {
@@ -182,7 +186,9 @@ public Function<ExpositionSink<?>, Boolean> visit(final SummaryMetricFamily metr
  writeMetric(buffer, metricFamily, "_count", summary.count, summary.labels);
 
  summary.quantiles.forEach(interval -> {
- writeMetric(buffer, metricFamily, null, interval.value, summary.labels, interval.quantile.asSummaryLabel());
+ if (!excludedHistoQuantiles.contains(interval.quantile)) {
+ writeMetric(buffer, metricFamily, null, interval.value, summary.labels, interval.quantile.asSummaryLabel());
+ }
  });
  });
  }
@@ -194,7 +200,10 @@ public Function<ExpositionSink<?>, Boolean> visit(final HistogramMetricFamily me
  writeMetric(buffer, metricFamily, "_count", histogram.count, histogram.labels);
 
  histogram.buckets.forEach(interval -> {
- writeMetric(buffer, metricFamily, "_bucket", interval.value, histogram.labels, interval.quantile.asHistogramLabel());
+ if (!excludedHistoQuantiles.contains(interval.quantile)) {
+ writeMetric(buffer, metricFamily, "_bucket", interval.value, histogram.labels, interval.quantile.asHistogramLabel()); 
+ }
+
  });
 
  writeMetric(buffer, metricFamily, "_bucket", histogram.count, histogram.labels, Interval.Quantile.POSITIVE_INFINITY.asHistogramLabel());

standalone/src/main/java/com/zegelin/cassandra/exporter/Application.java

@@ -6,7 +6,6 @@
 import com.datastax.driver.core.policies.RoundRobinPolicy;
 import com.datastax.driver.core.policies.WhiteListPolicy;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.zegelin.picocli.InetSocketAddressTypeConverter;
 import com.zegelin.picocli.JMXServiceURLTypeConverter;
 import com.zegelin.cassandra.exporter.cli.HarvesterOptions;
@@ -78,6 +77,9 @@ protected int defaultPort() {
 
  @Option(names = "--cql-password", paramLabel = "PASSWORD", description = "CQL authentication password.")
  private String cqlPassword;
+
+ @Option(names = "--cql-ssl", paramLabel = "SSL", description = "Creates enctrypted DB connections.")
+ private boolean ssl;
 
  @Option(names = {"-v", "--verbose"}, description = "Enable verbose logging. Multiple invocations increase the verbosity.")
  boolean[] verbosity = {};
@@ -145,6 +147,9 @@ private Cluster establishClusterConnection() {
  if (cqlUser != null && cqlPassword != null) {
  clusterBuilder.withCredentials(cqlUser, cqlPassword);
  }
+ if (ssl) {
+ clusterBuilder.withSSL();
+ }
 
  final Cluster cluster = clusterBuilder.build();