diff --git a/carbonj.service/Dockerfile b/carbonj.service/Dockerfile index 1d8c6916..c5b1d79a 100644 --- a/carbonj.service/Dockerfile +++ b/carbonj.service/Dockerfile @@ -18,8 +18,8 @@ RUN yum update -y && \ yum install -y gcc-c++ gcc make libtool automake autoconf make python3-devel && \ rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ yum install -y https://cdn.azul.com/zulu/bin/zulu-repo-1.0.0-1.noarch.rpm && \ - yum install -y https://mirror.stream.centos.org/9-stream/AppStream/$(uname -m)/os/Packages/pcp-conf-6.2.1-1.el9.$(uname -m).rpm && \ - yum install -y https://mirror.stream.centos.org/9-stream/AppStream/$(uname -m)/os/Packages/pcp-libs-6.2.1-1.el9.$(uname -m).rpm && \ + yum install -y https://mirror.stream.centos.org/9-stream/AppStream/$(uname -m)/os/Packages/pcp-conf-6.2.2-6.el9.$(uname -m).rpm && \ + yum install -y https://mirror.stream.centos.org/9-stream/AppStream/$(uname -m)/os/Packages/pcp-libs-6.2.2-6.el9.$(uname -m).rpm && \ # # If sysstat version is updated, confirm iolog.sh execution and update associated version check in entrypoint.sh # diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackMetric.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackMetric.java index 35d014da..4d70b707 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackMetric.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackMetric.java @@ -6,16 +6,6 @@ */ package com.demandware.carbonj.service.db.model; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import com.demandware.carbonj.service.accumulator.MetricAggregationPolicy; -import com.google.common.base.Preconditions; - -import com.demandware.carbonj.service.strings.StringsCache; - public class MsgPackMetric { final public String path; @@ -26,4 +16,4 @@ public MsgPackMetric( Metric metric ) this.path = metric.name; this.isLeaf = metric.isLeaf(); } -} \ No newline at end of file +} diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackSeries.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackSeries.java index 23adff32..7e828898 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackSeries.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackSeries.java @@ -7,7 +7,6 @@ package com.demandware.carbonj.service.db.model; import java.util.List; -import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonCreator; @@ -32,12 +31,7 @@ public class MsgPackSeries public MsgPackSeries( Series series) { - this.start = series.start; - this.end = series.end; - this.step = series.step; - this.name = series.name; - this.pathExpression = series.name; - this.values = series.values; + this(series.start, series.end, series.step, series.name, series.name, series.values); } @JsonCreator @@ -66,4 +60,4 @@ public String toString() ", values=" + values + '}'; } -} \ No newline at end of file +} diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/StorageAggregationPolicySource.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/StorageAggregationPolicySource.java index 54fc2f13..30f206ec 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/StorageAggregationPolicySource.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/StorageAggregationPolicySource.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,9 +25,9 @@ public class StorageAggregationPolicySource private static final Logger log = LoggerFactory.getLogger( StorageAggregationPolicySource.class ); // reuse same instance across multiple metrics. - private CopyOnWriteArrayList policies = new CopyOnWriteArrayList<>( ); + private final CopyOnWriteArrayList policies = new CopyOnWriteArrayList<>( ); - private StorageAggregationRulesLoader rulesLoader; + private final StorageAggregationRulesLoader rulesLoader; public StorageAggregationPolicySource( StorageAggregationRulesLoader rulesLoader) { @@ -75,15 +74,11 @@ public synchronized void cleanup() { log.info("checking for obsolete aggregation policies to remove from cache"); List obsolete = policies.stream() - .filter( p -> p.configChanged() ) - .collect( Collectors.toList()); + .filter(AggregationPolicy::configChanged) + .toList(); // no need to keep policies that represent obsolete config. policies.removeAll( obsolete ); log.info("purged obsolete aggregation policies from cache. Number of obsolete policies found: " + obsolete.size() + ", total number of policies after purge: " + policies.size()); } - - public StorageAggregationRulesLoader getRulesLoader() { - return rulesLoader; - } } diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/queue/InputQueue.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/queue/InputQueue.java index 61cc5880..70002e4b 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/queue/InputQueue.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/queue/InputQueue.java @@ -46,17 +46,15 @@ public class InputQueue extends Thread implements Consumer, StatsAware, Cl private final QueueProcessor pointProcessor; - private int batchSize; + private final int batchSize; - private BlockingQueue queue; + private final BlockingQueue queue; private volatile boolean stop = false; - private long emptyQueuePauseMillis; + private final long emptyQueuePauseMillis; - private RejectionHandler rh; - - private final int queueCapacity; + private final RejectionHandler rh; public InputQueue(MetricRegistry metricRegistry, String name, QueueProcessor queueProcessor, int queueSize, RejectionHandler rejectionHandler, int batchSize, @@ -65,7 +63,6 @@ public InputQueue(MetricRegistry metricRegistry, String name, QueueProcessor this.pointProcessor = Preconditions.checkNotNull(queueProcessor); this.batchSize = batchSize; this.emptyQueuePauseMillis = emptyQueuePauseMillis; - this.queueCapacity = queueSize; this.queue = new ArrayBlockingQueue<>( queueSize ); this.rh = rejectionHandler; @@ -100,13 +97,6 @@ public void drain() @Override public void run() { - - if ( queue == null ) - { - stop = true; - return; - } - // queue consumer loop. while ( true ) { @@ -202,19 +192,7 @@ public void refreshStats() public int queuedItemsCount() { - if ( queue == null ) - { - return 0; - } - else - { - return queue.size(); - } - } - - public int queueCapacity() - { - return queueCapacity; + return queue.size(); } private String queueSizeGaugeName(String name) diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/CountingLatePointLogger.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/CountingLatePointLogger.java new file mode 100644 index 00000000..16d31862 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/CountingLatePointLogger.java @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator; + +import com.demandware.carbonj.service.engine.DataPoint; + +import java.util.concurrent.atomic.AtomicInteger; + +public class CountingLatePointLogger implements LatePointLogger { + + private final AtomicInteger latePoints; + + public CountingLatePointLogger(AtomicInteger latePoints) { + this.latePoints = latePoints; + } + + @Override + public void logLatePoint(DataPoint m, long now, Reason r, String context) { + latePoints.incrementAndGet(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestAggregateFunction.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestAggregateFunction.java new file mode 100644 index 00000000..b97c1d6c --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestAggregateFunction.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestAggregateFunction { + @Test + public void testAggregateFunction() { + AggregateFunction aggregateFunction = AggregateFunction.create("foo.bar.p95", MetricAggregationMethod.CUSTOM1); + assertInstanceOf(AggregateFunction.AvgAggregateFunction.class, aggregateFunction); + assertEquals(0, aggregateFunction.apply()); + try { + aggregateFunction.getValues(); + fail("Should have thrown an exception"); + } catch (UnsupportedOperationException e) { + } + aggregateFunction = AggregateFunction.create("foo.bar.latency", MetricAggregationMethod.LATENCY); + assertInstanceOf(AggregateFunction.LatencyAggregateFunction.class, aggregateFunction); + assertEquals(0, aggregateFunction.apply()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestLatePointLoggerImpl.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestLatePointLoggerImpl.java new file mode 100644 index 00000000..a4807066 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestLatePointLoggerImpl.java @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.engine.DataPoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestLatePointLoggerImpl { + @Test + public void test() { + MetricRegistry metricRegistry = new MetricRegistry(); + LatePointLoggerImpl latePointLogger = new LatePointLoggerImpl(metricRegistry); + latePointLogger.logLatePoint(new DataPoint("foo.bar", 123, 60), 120, LatePointLogger.Reason.SLOT_CLOSED, "Context"); + assertEquals(1, metricRegistry.getCounters().get("aggregator.skippedDelayed").getCount()); + assertEquals(60, metricRegistry.getHistograms().get("aggregator.pointAgeHistogram").getSnapshot().get95thPercentile()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestMetricAggregationRule.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestMetricAggregationRule.java new file mode 100644 index 00000000..42ec875d --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestMetricAggregationRule.java @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestMetricAggregationRule { + @Test + public void test() { + String aggregationRule = "ocapi... (60) c = custom1 pod[0-9]{3,6}.ecom...*.*.ocapi.clients.*.<>"; + MetricAggregationRule metricAggregationRule = MetricAggregationRule.parseDefinition(aggregationRule, 1, false); + assertFalse(metricAggregationRule.isStopRule()); + assertEquals(MetricAggregationMethod.CUSTOM1, metricAggregationRule.getMethod()); + assertTrue(metricAggregationRule.equals(metricAggregationRule)); + assertFalse(metricAggregationRule.equals(new Object())); + assertTrue(metricAggregationRule.hashCode() > 0); + assertEquals(String.format("com.demandware.carbonj.service.accumulator.MetricAggregationRule@%x", System.identityHashCode(metricAggregationRule)), metricAggregationRule.toString()); + assertFalse(metricAggregationRule.equals(MetricAggregationRule.parseDefinition( + "ocapi... (60) c = sum pod[0-9]{3,6}.ecom...*.*.ocapi.clients.*.<>", 1, false))); + MetricAggregationRule.Result result = metricAggregationRule.apply("pod807.ecom.bgzz.bgzz_prd.blade_1.bgzz_prd.ocapi.clients.client.foo.bar"); + assertTrue(result.equals(result)); + assertFalse(result.equals(null)); + assertNotEquals(0, result.hashCode()); + assertEquals("Result{aggregateName=ocapi.bgzz.bgzz_prd.foo.bar, method=CUSTOM1, dropOriginal=false}", result.toString()); + assertFalse(result.equals(metricAggregationRule.apply("pod807.ecom.bgzz.bgzz_prd.blade_1.bgzz_prd.ocapi.clients.client.foo.bar2"))); + + String aggregationRule2 = "ocapi... (60) cc = custom1 pod[0-9]{3,6}.ecom...*.*.ocapi.clients.*.<>"; + try { + MetricAggregationRule.parseDefinition(aggregationRule2, 2, false); + fail("Should have thrown an exception"); + } catch (RuntimeException e) { + assertEquals("Unsupported flag: [cc]", e.getMessage()); + } + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestMetricAggregationRules.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestMetricAggregationRules.java new file mode 100644 index 00000000..0ab016b5 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestMetricAggregationRules.java @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestMetricAggregationRules { + @Test + public void test() { + MetricAggregationRules metricAggregationRules = new MetricAggregationRules(1, new ArrayList<>()); + assertTrue(metricAggregationRules.isEmpty()); + assertEquals("MetricAggregationRules{revision=1, rules=[]}", metricAggregationRules.toString()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestSlot.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestSlot.java new file mode 100644 index 00000000..b238f268 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/TestSlot.java @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.demandware.carbonj.service.engine.DataPoint; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestSlot { + @Test + public void testSlot() { + MetricRegistry metricRegistry = new MetricRegistry(); + Timer aggregatorFlushTimer = metricRegistry.timer(MetricRegistry.name("aggregator", "slotFlushTimer")); + Meter flushedAggregates = metricRegistry.meter(MetricRegistry.name("aggregator", "aggregates")); + Meter createdSlots = metricRegistry.meter(MetricRegistry.name("aggregator", "slotCreated")); + AtomicInteger counter = new AtomicInteger(); + Slot slot = new Slot(60, new CountingLatePointLogger(counter), 1, aggregatorFlushTimer, flushedAggregates, createdSlots); + MetricAggregate metricAggregate = new MetricAggregate("foo.bar.aggregated", MetricAggregationMethod.LATENCY, false); + slot.apply(metricAggregate, new DataPoint("foo.bar", 123, 90, false), 90); + assertEquals(1, slot.size()); + assertEquals(60, slot.getTs()); + slot.close(dataPoints -> {System.out.println(dataPoints.get(0));}); + assertEquals(4, flushedAggregates.getCount()); + assertEquals(0, counter.get()); + slot.apply(metricAggregate, new DataPoint("foo.bar", 123, 90, false), 90); + assertEquals(1, counter.get()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryAccumulator.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryAccumulator.java index 3dac100a..ee8fa2d3 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryAccumulator.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryAccumulator.java @@ -7,8 +7,8 @@ package com.demandware.carbonj.service.accumulator.recovery; import com.demandware.carbonj.service.BaseTest; +import com.demandware.carbonj.service.accumulator.CountingLatePointLogger; import com.demandware.carbonj.service.accumulator.DefaultSlotStrategy; -import com.demandware.carbonj.service.accumulator.LatePointLogger; import com.demandware.carbonj.service.accumulator.MetricAggregate; import com.demandware.carbonj.service.accumulator.MetricAggregationMethod; import com.demandware.carbonj.service.accumulator.MetricAggregationPolicy; @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -75,6 +76,13 @@ public void testAllRecoveryPoints() { assertEquals(0, accumulator.getTimeSlots().size()); assertEquals(4, latePoints.get()); + + assertEquals(240, accumulator.getMaxClosedSlotTs()); + assertInstanceOf(DefaultSlotStrategy.class, accumulator.getSlotStrategy()); + accumulator.refreshStats(); + accumulator.dumpStats(); + accumulator.reset(); + assertEquals(0, accumulator.getMaxClosedSlotTs()); } private long toMillis(int sec) { @@ -125,18 +133,4 @@ public void testLatePoints() { private DataPoint getDataPoint(int ts) { return new DataPoint("metric", 1, ts); } - - private static class CountingLatePointLogger implements LatePointLogger { - - private final AtomicInteger latePoints; - - private CountingLatePointLogger(AtomicInteger latePoints) { - this.latePoints = latePoints; - } - - @Override - public void logLatePoint(DataPoint m, long now, Reason r, String context) { - latePoints.incrementAndGet(); - } - } } diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryLatePointLogger.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryLatePointLogger.java new file mode 100644 index 00000000..faa23d71 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/accumulator/recovery/TestRecoveryLatePointLogger.java @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.accumulator.recovery; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.accumulator.LatePointLogger; +import com.demandware.carbonj.service.engine.DataPoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestRecoveryLatePointLogger { + @Test + public void testDefault() { + MetricRegistry metricRegistry = new MetricRegistry(); + RecoveryLatePointLogger logger = new RecoveryLatePointLogger(metricRegistry); + int current = (int) (System.currentTimeMillis()/ 1000); + DataPoint dataPoint = new DataPoint("foo.bar", 123, current); + logger.logLatePoint(dataPoint, current, LatePointLogger.Reason.SLOT_EXPIRED, "test"); + assertEquals(1, metricRegistry.getCounters().get("aggregator.recoveryDelayed").getCount()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/admin/_DumpFormat.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/admin/_DumpFormat.java index 61268d92..230a6c71 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/admin/_DumpFormat.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/admin/_DumpFormat.java @@ -20,8 +20,10 @@ public class _DumpFormat @Test public void writeSeries() { + DataPointValue dataPointValue = new DataPointValue(100, 2.0f); + assertEquals("100 2", dataPointValue.toString()); assertEquals( "abc|100|1|2", - DumpFormat.writeSeries( "abc", 1, List.of(new DataPointValue(100, 2.0f))) ); + DumpFormat.writeSeries( "abc", 1, List.of(dataPointValue))); assertEquals( "abc|100|1|2.35", DumpFormat.writeSeries( "abc", 1, List.of(new DataPointValue(100, 2.351f))) ); assertEquals( diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/TestQueryDurations.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/TestQueryDurations.java new file mode 100644 index 00000000..b7db2ac7 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/TestQueryDurations.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestQueryDurations { + @Test + public void testQueryDurations() { + QueryDurations queryDurations = new QueryDurations(); + queryDurations.addRead(1); + assertEquals(1, queryDurations.read()); + queryDurations.addSerializeAndSend(2); + assertEquals(2, queryDurations.serializeAndSend()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/TestStreamSeriesBatchMetricsTask.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/TestStreamSeriesBatchMetricsTask.java new file mode 100644 index 00000000..4a74d5b1 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/TestStreamSeriesBatchMetricsTask.java @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.db.model.*; +import com.demandware.carbonj.service.db.util.DatabaseMetrics; +import com.demandware.carbonj.service.engine.DataPoint; +import com.demandware.carbonj.service.engine.DataPoints; +import com.demandware.carbonj.service.engine.GraphitePickler; +import com.demandware.carbonj.service.engine.Query; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStreamSeriesBatchMetricsTask { + @Test + public void test() throws Exception { + MetricRegistry metricRegistry = new MetricRegistry(); + DatabaseMetrics databaseMetrics = new DatabaseMetrics(metricRegistry); + Metric metric = new Metric("foo.bar", 1, null, List.of(RetentionPolicy.getInstance("60s:24h")), new ArrayList<>()); + StreamSeriesBatchMetricsTask streamSeriesBatchMetricsTask = new StreamSeriesBatchMetricsTask( + new MockDataPointStore(), List.of(metric), new Query("foo.bar", 0, 60, 0, 1000), new GraphitePickler(System.out), new QueryDurations()); + BatchStats batchStats = streamSeriesBatchMetricsTask.call(); + assertEquals(2, batchStats.noOfDataPoints); + assertEquals(String.format("Task [%d] for pattern [foo.bar], from=0, until=60, now=0, size=1", streamSeriesBatchMetricsTask.hashCode()), streamSeriesBatchMetricsTask.toString()); + } + + private static class MockDataPointStore implements DataPointStore { + + @Override + public DataPointImportResults importDataPoints(String dbName, List dataPoint, int maxAllowedImportErrors) { + return null; + } + + @Override + public void insertDataPoints(DataPoints points) { + + } + + @Override + public void importDataPoints(String dbName, DataPoints points) { + + } + + @Override + public long delete(String archive, int ts) { + return 0; + } + + @Override + public void delete(List m) { + + } + + @Override + public Series getSeries(Metric metric, int from, int until, int now) { + return new Series("foo.bar", from, until, 60, List.of(1.0, 2.0)); + } + + @Override + public List getValues(RetentionPolicy archivePolicy, long metricId, int from, int to) { + return List.of(); + } + + @Override + public void open() { + + } + + @Override + public void close() { + + } + + @Override + public DataPointValue getFirst(RetentionPolicy instanceForDbName, long id, int from, int to) { + return null; + } + + @Override + public void dumpStats() { + + } + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/log/TestQueryStats.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/log/TestQueryStats.java new file mode 100644 index 00000000..0ad5e1e8 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/log/TestQueryStats.java @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.log; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestQueryStats { + @Test + public void testDefault() { + QueryStats queryStats = new QueryStats(); + assertEquals(0, queryStats.getWaitTimeMillis().getCount()); + assertEquals(0, queryStats.getSeriesReadTimeMillis().getCount()); + assertEquals(0, queryStats.getSeriesWriteTimeMillis().getCount()); + assertEquals(0, queryStats.getEmptySeriesReadTimeMillis().getCount()); + assertEquals(0, queryStats.getTotalNoOfDataPoints()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/RetentionPolicyTest.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/RetentionPolicyTest.java index 1ee6e4fb..5e9f3d77 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/RetentionPolicyTest.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/RetentionPolicyTest.java @@ -12,6 +12,9 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class RetentionPolicyTest { @@ -24,4 +27,27 @@ public void testParseLine() RetentionPolicy.getInstance( "30m:2y")); assertEquals(RetentionPolicy.getPolicyList( "60s:24h,5m:7d,30m:2y" ), expected); } + + @Test + public void testNegatives() { + try { + RetentionPolicy.getInstance("5t"); + fail("Should have thrown an exception"); + } catch (IllegalArgumentException e) { + assertEquals("Unsupported time unit suffix [t] in value [5t]", e.getMessage()); + } + RetentionPolicy retentionPolicy = RetentionPolicy.getInstance("60s:1h"); + try { + retentionPolicy.assertTimestampMatchesThisPolicyInterval(1); + fail("Should have thrown an exception"); + } catch (RuntimeException e) { + assertEquals("timestamp does not match any interval from this retention policy. ts: 1, retention policy interval for ts: 0", e.getMessage()); + } + assertEquals(retentionPolicy, RetentionPolicy.higherPrecision(retentionPolicy, null)); + assertTrue(retentionPolicy.equals(RetentionPolicy.getInstance("60s:1h"))); + assertFalse(retentionPolicy.equals(new Object())); + assertFalse(retentionPolicy.equals(RetentionPolicy.getInstance("60s:24h"))); + assertEquals("RetentionPolicy{name='60s:1h', precision=60, retention=3600}", retentionPolicy.toString()); + assertEquals(0, retentionPolicy.maxPoints(0, 0, 3601)); + } } diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestAggregationPolicy.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestAggregationPolicy.java new file mode 100644 index 00000000..657c17dd --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestAggregationPolicy.java @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import com.demandware.carbonj.service.engine.AggregationMethod; +import com.demandware.carbonj.service.engine.StorageAggregationRulesLoader; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestAggregationPolicy { + @Test + public void test() { + File storageConfFile = new File(Objects.requireNonNull(this.getClass().getClassLoader().getResource("storage-aggregation.conf")).getFile()); + AggregationPolicy aggregationPolicy = new AggregationPolicy(AggregationMethod.AVG, 0, + new StorageAggregationPolicySource(new StorageAggregationRulesLoader(storageConfFile))); + assertTrue(aggregationPolicy.configChanged()); + assertEquals(AggregationMethod.AVG, aggregationPolicy.getInstance("foo.bar").getMethod()); + assertEquals(1.5, aggregationPolicy.apply(List.of(1.0, 2.0)).getAsDouble()); + assertTrue(aggregationPolicy.equals(aggregationPolicy)); + assertFalse(aggregationPolicy.equals(new Object())); + assertTrue(aggregationPolicy.getInstance("foo.bar").equals(aggregationPolicy.getInstance("foo.bar2"))); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestDataPointImportResults.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestDataPointImportResults.java new file mode 100644 index 00000000..4740d848 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestDataPointImportResults.java @@ -0,0 +1,16 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +public class TestDataPointImportResults { + @Test + public void test() { + DataPointImportResults dataPointImportResults = new DataPointImportResults("60s24h", 0, 0, 0, 0); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestDeleteAPIResult.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestDeleteAPIResult.java new file mode 100644 index 00000000..0c2ceb24 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestDeleteAPIResult.java @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestDeleteAPIResult { + @Test + public void test() { + DeleteAPIResult deleteAPIResult = new DeleteAPIResult(); + deleteAPIResult.setResponseTtime(1); + assertEquals(1, deleteAPIResult.getResponseTtime()); + deleteAPIResult.setSuccess(true); + assertTrue(deleteAPIResult.getSuccess()); + deleteAPIResult.setError("error"); + assertEquals("error", deleteAPIResult.getError()); + assertTrue(deleteAPIResult.getMetricsList().isEmpty()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestInterval.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestInterval.java new file mode 100644 index 00000000..770aea38 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestInterval.java @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestInterval { + @Test + public void testInterval() { + Interval interval = new Interval(0, 60); + assertTrue(interval.equals(interval)); + assertFalse(interval.equals(new Object())); + assertFalse(interval.equals(new Interval(1, 1))); + assertTrue(interval.equals(new Interval(0, 60))); + assertTrue(interval.hashCode() > 0); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestIntervalValues.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestIntervalValues.java new file mode 100644 index 00000000..10c6385f --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestIntervalValues.java @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import com.demandware.carbonj.service.engine.AggregationMethod; +import com.demandware.carbonj.service.engine.DataPoint; +import com.demandware.carbonj.service.engine.StorageAggregationRulesLoader; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class TestIntervalValues { + @Test + public void testIntervalValues() { + Metric m = new Metric( "a.b", 1, null, new ArrayList<>(), null ); + assertFalse(m.isLeaf()); + IntervalValues intervalValues = new IntervalValues(m, new ArrayList<>(), 0, "60s24h"); + assertNull(intervalValues.toDataPoint()); + List retentionPolicyList = new ArrayList<>(); + retentionPolicyList.add(RetentionPolicy.getInstance("60s:24h")); + File storageConfFile = new File(Objects.requireNonNull(this.getClass().getClassLoader().getResource("storage-aggregation.conf")).getFile()); + AggregationPolicy aggregationPolicy = new AggregationPolicy(AggregationMethod.AVG, 0, + new StorageAggregationPolicySource(new StorageAggregationRulesLoader(storageConfFile))); + m = new Metric( "a.b", 1, aggregationPolicy, retentionPolicyList, null ); + intervalValues = new IntervalValues(m, List.of(1.0, 2.0), 120, "60s24h"); + DataPoint dataPoint = intervalValues.toDataPoint(); + assertNotNull(dataPoint); + assertEquals("a.b", dataPoint.name); + assertEquals(120, dataPoint.ts); + assertEquals(1.5, dataPoint.val); + assertEquals("IntervalValues{metric=Metric{id=1, name='a.b'}, intervalStart=120, dbName='60s24h', values=[1.0, 2.0]}", intervalValues.toString()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestIntervals.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestIntervals.java new file mode 100644 index 00000000..b75bb26c --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestIntervals.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import com.demandware.carbonj.service.engine.AggregationMethod; +import com.demandware.carbonj.service.engine.DataPoints; +import com.demandware.carbonj.service.engine.StorageAggregationRulesLoader; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestIntervals { + @Test + public void testIntervals() { + File storageConfFile = new File(Objects.requireNonNull(this.getClass().getClassLoader().getResource("storage-aggregation.conf")).getFile()); + StorageAggregationPolicySource storageAggregationPolicySource = new StorageAggregationPolicySource( + new StorageAggregationRulesLoader(storageConfFile)); + AggregationPolicy aggregationPolicy1 = new AggregationPolicy(AggregationMethod.AVG, 1, storageAggregationPolicySource); + Metric metric1 = new Metric("foo.bar1", 1, aggregationPolicy1, List.of(RetentionPolicy.getInstance("60s:24h")), new ArrayList<>()); + IntervalValues intervalValues1 = new IntervalValues(metric1, List.of(1.0, 2.0), 0, "60s24h"); + AggregationPolicy aggregationPolicy2 = new AggregationPolicy(AggregationMethod.SUM, 1, storageAggregationPolicySource); + Metric metric2 = new Metric("foo.bar2", 2, aggregationPolicy2, List.of(RetentionPolicy.getInstance("60s:24h")), new ArrayList<>()); + IntervalValues intervalValues2 = new IntervalValues(metric2, List.of(3.0, 4.0), 0, "60s24h"); + Intervals intervals = new Intervals(List.of(intervalValues1, intervalValues2)); + DataPoints dataPoints = intervals.toDataPoints(); + assertEquals(2, dataPoints.size()); + assertEquals("foo.bar1", dataPoints.get(0).name); + assertEquals("foo.bar2", dataPoints.get(1).name); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestMsgPackMetric.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestMsgPackMetric.java new file mode 100644 index 00000000..c43359ea --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestMsgPackMetric.java @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestMsgPackMetric { + @Test + public void test() { + Metric metric = new Metric("foo.bar", 1, null, List.of(RetentionPolicy.getInstance("60s:24h")), new ArrayList<>()); + MsgPackMetric msgPackMetric = new MsgPackMetric(metric); + assertEquals(NullMetric.getInstance(), NullMetric.METRIC_NULL); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestMsgPackSeries.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestMsgPackSeries.java new file mode 100644 index 00000000..fa1011b3 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestMsgPackSeries.java @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestMsgPackSeries { + @Test + public void testSeries() { + Series series = new Series("name", 0, 60, 60, List.of(1.0, 2.0)); + assertEquals("Series{name='name', start=0, end=60, step=60, values=[1.0, 2.0]}", series.toString()); + MsgPackSeries msgPackSeries = new MsgPackSeries(series); + assertEquals("MsgPackSeries{start=0, end=60, step=60, name=name, pathExpression=name, values=[1.0, 2.0]}", msgPackSeries.toString()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestStorageAggregationPolicySource.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestStorageAggregationPolicySource.java new file mode 100644 index 00000000..1b4947c4 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestStorageAggregationPolicySource.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import com.demandware.carbonj.service.engine.AggregationMethod; +import com.demandware.carbonj.service.engine.StorageAggregationRulesLoader; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStorageAggregationPolicySource { + @Test + public void testStorageAggregationPolicySource() { + String metricName = "pod276.ecom.bbdl.bbdl_prd.blade6-4.bbdl_prd.ocapi.clients.99c874ec-90ef-42fa-bf08-4becc2893202.apis.shop.versions.v20_2.methods.post.resources.customers_auth.requests.m15_rate"; + File storageConfFile = new File(Objects.requireNonNull(this.getClass().getClassLoader().getResource("storage-aggregation.conf")).getFile()); + StorageAggregationRulesLoader storageAggregationRulesLoader = new StorageAggregationRulesLoader(storageConfFile); + StorageAggregationPolicySource storageAggregationPolicySource = new StorageAggregationPolicySource(storageAggregationRulesLoader); + assertEquals(1, storageAggregationPolicySource.currentConfigRevision()); + AggregationPolicy aggregationPolicy = storageAggregationPolicySource.policyForMetricName(metricName); + assertEquals(AggregationMethod.AVG, aggregationPolicy.getMethod()); + storageAggregationPolicySource.cleanup(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestTooManyDatapointsFoundException.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestTooManyDatapointsFoundException.java new file mode 100644 index 00000000..ab26464e --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestTooManyDatapointsFoundException.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestTooManyDatapointsFoundException { + @Test + public void test() { + TooManyDatapointsFoundException exception1 = new TooManyDatapointsFoundException(1); + assertEquals(1, exception1.getLimit()); + TooManyDatapointsFoundException exception2 = new TooManyDatapointsFoundException(2, "error"); + assertEquals(2, exception2.getLimit()); + assertEquals("error", exception2.getMessage()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestTooManyMetricsFoundException.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestTooManyMetricsFoundException.java new file mode 100644 index 00000000..bd1ea059 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/TestTooManyMetricsFoundException.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.model; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestTooManyMetricsFoundException { + @Test + public void test() { + TooManyMetricsFoundException exception1 = new TooManyMetricsFoundException(1); + assertEquals(1, exception1.getLimit()); + TooManyMetricsFoundException exception2 = new TooManyMetricsFoundException(2, "error"); + assertEquals(2, exception2.getLimit()); + assertEquals("error", exception2.getMessage()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/_Metric.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/_Metric.java index 1a8461cb..048ac282 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/_Metric.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/model/_Metric.java @@ -6,15 +6,19 @@ */ package com.demandware.carbonj.service.db.model; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import com.demandware.carbonj.service.db.util.time.TimeSource; +import com.demandware.carbonj.service.engine.AggregationMethod; +import com.demandware.carbonj.service.engine.StorageAggregationRulesLoader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; public class _Metric { @@ -43,4 +47,20 @@ public void shouldReturnMaxRetentionIntervalForLeafMetric() int expected = 2 * 365 * 24 * 60 * 60; // 2years in seconds. assertEquals(interval.end - interval.start, expected); } + + @Test + public void testNegatives() { + File storageConfFile = new File(Objects.requireNonNull(this.getClass().getClassLoader().getResource("storage-aggregation.conf")).getFile()); + AggregationPolicy aggregationPolicy = new AggregationPolicy(AggregationMethod.AVG, 0, + new StorageAggregationPolicySource(new StorageAggregationRulesLoader(storageConfFile))); + Metric m = new Metric( "a.b", 1, null, new ArrayList<>(), null ); + assertTrue(m.retentionPolicyAfter(null).isEmpty()); + assertEquals(0, m.getMaxRetention()); + m = new Metric( "a.b", 1, aggregationPolicy, null, null ); + assertTrue(m.getHighestPrecisionArchive().isEmpty()); + assertTrue(m.pickArchiveForQuery(0, 0, 0).isEmpty()); + assertFalse(m.getAggregationPolicy().configChanged()); + assertFalse(m.equals(new Object())); + assertEquals("Metric{id=1, name='a.b'}", m.toString()); + } } diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/DataPointStoreUtils.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/DataPointStoreUtils.java index c1883312..6b05e006 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/DataPointStoreUtils.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/DataPointStoreUtils.java @@ -20,7 +20,9 @@ public class DataPointStoreUtils { public static DataPointStore createDataPointStore(MetricRegistry metricRegistry, File dbDirFile, boolean longId, MetricIndex metricIndex) { File stagingDir = new File(dbDirFile, "staging"); - assertTrue(stagingDir.mkdirs()); + if (!stagingDir.exists()) { + assertTrue(stagingDir.mkdirs()); + } StagingFiles sFiles = new StagingFiles(metricRegistry, stagingDir, new SystemSort(), metricIndex); DataPointArchiveFactory pointArchiveFactory = new DataPointArchiveFactory(metricRegistry, dbDirFile, new RocksDBConfig(), longId); DataPointStagingStore stagingStore = new DataPointStagingStore( metricRegistry, sFiles, 1000, 1, 1, 1, 100, 30, 3); diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointArchiveRocksDB.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointArchiveRocksDB.java new file mode 100644 index 00000000..99911671 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointArchiveRocksDB.java @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.db.model.DataPointValue; +import com.demandware.carbonj.service.db.model.RetentionPolicy; +import com.demandware.carbonj.service.engine.DataPoint; +import com.demandware.carbonj.service.engine.DataPoints; +import org.junit.jupiter.api.Test; +import org.rocksdb.RocksDBException; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestDataPointArchiveRocksDB { + @Test + public void test() { + MetricRegistry metricRegistry = new MetricRegistry(); + RocksDBConfig rocksDBConfig = new RocksDBConfig(); + DataPointArchiveRocksDB dataPointArchiveRocksDB = new DataPointArchiveRocksDB( + metricRegistry, "60s24h", RetentionPolicy.getInstance("60s:24h"), new File("/tmp/testdb"), rocksDBConfig, true); + dataPointArchiveRocksDB.open(); + dataPointArchiveRocksDB.put(12345L, 60, 123.45); + List dataPointValueList = dataPointArchiveRocksDB.getDataPoints(12345L, 0, 60); + assertEquals(1, dataPointValueList.size()); + DataPointValue dataPointValue = dataPointValueList.get(0); + assertEquals(60, dataPointValue.ts); + assertEquals(123.45, dataPointValue.val); + dataPointArchiveRocksDB.delete(60); + dataPointValueList = dataPointArchiveRocksDB.getDataPoints(12345L, 0, 60); + assertTrue(dataPointValueList.isEmpty()); + DataPoint dataPoint = new DataPoint("foo.bar", 123, 0); + assertEquals(0, dataPointArchiveRocksDB.put(new DataPoints(List.of(dataPoint)))); + dataPoint = new DataPoint("foo.bar", 123, 60); + dataPoint.setMetricId(DataPoint.UNKNOWN_ID); + assertEquals(0, dataPointArchiveRocksDB.put(new DataPoints(List.of(dataPoint)))); + dataPoint.setMetricId(12345L); + assertEquals(0, dataPointArchiveRocksDB.put(new DataPoints(List.of(dataPoint)))); + dataPoint = new DataPoint("foo.bar", 123, (int) (System.currentTimeMillis() / 1000)); + dataPoint.setMetricId(12345L); + assertEquals(0, dataPointArchiveRocksDB.put(new DataPoints(List.of(dataPoint)))); + dataPointArchiveRocksDB.close(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointRecord.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointRecord.java new file mode 100644 index 00000000..87b1806e --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointRecord.java @@ -0,0 +1,21 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestDataPointRecord { + @Test + public void test() { + byte[] bytes = DataPointRecord.toKeyBytes(2, 60, true); + assertEquals(2, DataPointRecord.toMetricId(bytes, true)); + bytes = DataPointRecord.toKeyBytes(3, 60, false); + assertEquals(3, DataPointRecord.toMetricId(bytes, false)); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointStoreImpl.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointStoreImpl.java new file mode 100644 index 00000000..077bdb0f --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestDataPointStoreImpl.java @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.db.model.DataPointImportResults; +import com.demandware.carbonj.service.db.model.DataPointStore; +import com.demandware.carbonj.service.db.model.Metric; +import com.demandware.carbonj.service.db.model.QueryCachePolicy; +import com.demandware.carbonj.service.db.model.RetentionPolicy; +import com.demandware.carbonj.service.db.model.Series; +import com.demandware.carbonj.service.db.util.DatabaseMetrics; +import com.demandware.carbonj.service.engine.DataPoint; +import com.demandware.carbonj.service.engine.DataPoints; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestDataPointStoreImpl { + @Test + public void test() { + MetricRegistry metricRegistry = new MetricRegistry(); + RocksDBConfig rocksDBConfig = new RocksDBConfig(); + DatabaseMetrics databaseMetrics = new DatabaseMetrics(metricRegistry); + DataPointArchiveFactory dataPointArchiveFactory = new DataPointArchiveFactory(metricRegistry, new File("/tmp/testdb"), rocksDBConfig, true); + DataPointStore dataPointStore = new DataPointStoreImpl(metricRegistry, dataPointArchiveFactory, databaseMetrics, null, + true, 1, 0, + new QueryCachePolicy(false, false, false, false), new Predicate() { + @Override + public boolean test(String s) { + return !s.endsWith(".invalid"); + } + }); + RetentionPolicy retentionPolicy = RetentionPolicy.getInstance("60s:24h"); + DataPoint dataPoint = new DataPoint("foo.bar", 123, 60); + dataPoint.setMetricId(12345L); + DataPointImportResults dataPointImportResults = dataPointStore.importDataPoints("60s24h", List.of(dataPoint), 0); + assertEquals(1, dataPointImportResults.received); + assertEquals(1, dataPointImportResults.expired); + + dataPoint = new DataPoint("foo.bar", 123, 61); + dataPoint.setMetricId(12345L); + try { + dataPointStore.importDataPoints("60s24h", List.of(dataPoint), 0); + fail("Should have thrown an exception"); + } catch (RuntimeException e) { + assertEquals("Number of errors [1] exceeds specified maxAllowedImportErrors [0] for the request", e.getMessage()); + } + + int current = (int) (System.currentTimeMillis() / 1000); + dataPoint = new DataPoint("foo.bar", 123, retentionPolicy.interval(current)); + dataPoint.setMetricId(12345L); + dataPointImportResults = dataPointStore.importDataPoints("60s24h", List.of(dataPoint), 0); + assertEquals(1, dataPointImportResults.received); + assertEquals(1, dataPointImportResults.saved); + + Metric metric = new Metric("foo.bar", 12345L, null, List.of(retentionPolicy), new ArrayList<>()); + Series series = dataPointStore.getSeries(metric, retentionPolicy.interval(current), retentionPolicy.interval(current) + 60, (int) (System.currentTimeMillis() / 1000)); + assertEquals(2, series.values.size()); + assertEquals(123, series.values.get(0)); + assertNull(series.values.get(1)); + + assertEquals(1, dataPointStore.delete("60s24h", retentionPolicy.interval(current))); + + metric = new Metric("foo.bar.invalid", 12345L, null, List.of(retentionPolicy), new ArrayList<>()); + series = dataPointStore.getSeries(metric, retentionPolicy.interval(current), retentionPolicy.interval(current) + 60, (int) (System.currentTimeMillis() / 1000)); + assertEquals(2, series.values.size()); + assertNull(series.values.get(0)); + assertNull(series.values.get(1)); + + // Negative tests + dataPoint = new DataPoint("foo.bar", 123, 0); + DataPoints dataPoints = new DataPoints(List.of(dataPoint)); + metric = new Metric("foo.bar", 12345L, null, List.of(retentionPolicy), new ArrayList<>()); + dataPoints.assignMetric(0, metric, retentionPolicy); + dataPointStore.insertDataPoints(dataPoints); + + dataPoint = new DataPoint("foo.bar", 123, 60); + dataPoint.setMetricId(DataPoint.UNKNOWN_ID); + dataPoints = new DataPoints(List.of(dataPoint)); + metric = new Metric("foo.bar", DataPoint.UNKNOWN_ID, null, List.of(retentionPolicy), new ArrayList<>()); + dataPoints.assignMetric(0, metric, retentionPolicy); + dataPointStore.insertDataPoints(dataPoints); + + dataPointStore.close(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestExpiredOnArrivalDataPointException.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestExpiredOnArrivalDataPointException.java new file mode 100644 index 00000000..377e66c5 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestExpiredOnArrivalDataPointException.java @@ -0,0 +1,27 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import com.demandware.carbonj.service.db.model.RetentionPolicy; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestExpiredOnArrivalDataPointException { + @Test + public void test() { + ExpiredOnArrivalDataPointException exception = new ExpiredOnArrivalDataPointException(60, 123, RetentionPolicy.getInstance("60s:24h")); + assertNotEquals(0, exception.hashCode()); + assertTrue(exception.equals(exception)); + assertFalse(exception.equals(new Object())); + assertFalse(exception.equals(new ExpiredOnArrivalDataPointException(120, 123, RetentionPolicy.getInstance("60s:24h")))); + assertFalse(exception.equals(new ExpiredOnArrivalDataPointException(60, 1234, RetentionPolicy.getInstance("60s:24h")))); + assertTrue(exception.equals(new ExpiredOnArrivalDataPointException(60, 123, RetentionPolicy.getInstance("60s:24h")))); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessorImpl.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessorImpl.java new file mode 100644 index 00000000..e70a91d1 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessorImpl.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.db.index.IndexUtils; +import com.demandware.carbonj.service.db.model.DataPointStore; +import com.demandware.carbonj.service.db.model.IntervalValues; +import com.demandware.carbonj.service.db.model.Metric; +import com.demandware.carbonj.service.db.model.MetricIndex; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestIntervalProcessorImpl { + @Test + public void test() throws Exception { + MetricRegistry metricRegistry = new MetricRegistry(); + MetricIndex metricIndex = IndexUtils.metricIndex(new File("/tmp/testdb"), true); + metricIndex.open(); + Metric metric1 = metricIndex.createLeafMetric("pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io"); + assertEquals(2, metric1.id); + DataPointStore dataPointStore = DataPointStoreUtils.createDataPointStore(metricRegistry, new File("/tmp/testdb"), true, metricIndex); + IntervalProcessorImpl intervalProcessor = new IntervalProcessorImpl(metricRegistry, "60s24h", 1, 1, 1, new IntervalProcessorTaskFactoryImpl(dataPointStore), 100); + assertEquals("60s24h", intervalProcessor.getDbName()); + assertEquals("IntervalProcessor{, dbName='60s24h'}", intervalProcessor.toString()); + Thread thread = new Thread(intervalProcessor); + thread.start(); + IntervalValues intervalValues = new IntervalValues(metric1, List.of(1.0, 2.0), 60, "60s24h"); + intervalProcessor.put(intervalValues); + Thread.sleep(500); + intervalProcessor.close(); + thread.join(); + metricIndex.close(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessors.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessors.java index 7789703e..de5a55a8 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessors.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestIntervalProcessors.java @@ -7,8 +7,10 @@ package com.demandware.carbonj.service.db.points; import com.demandware.carbonj.service.BaseTest; +import com.demandware.carbonj.service.db.index.IndexUtils; import com.demandware.carbonj.service.db.model.*; import com.demandware.carbonj.service.engine.AggregationMethod; +import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -75,4 +77,29 @@ public Metric forName(String metricName) { Collections.emptyList()); } } + + @Test + public void test() throws Exception { + MetricIndex metricIndex = IndexUtils.metricIndex(new File("/tmp/testdb"), true); + metricIndex.open(); + Metric metric1 = metricIndex.createLeafMetric("pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io"); + assertEquals(2, metric1.id); + Metric metric2 = metricIndex.createLeafMetric("POD276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.a2su9qja883gs.1965276335.sqlmonitor.elapsed_time"); + assertEquals(3, metric2.id); + DataPointStore dataPointStore = DataPointStoreUtils.createDataPointStore(metricRegistry, new File("/tmp/testdb"), true, metricIndex); + File staging = new File("/tmp/5m7d-1734989700-9.1.s"); + List lines = List.of( + "11987976699 2 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", + "11987976699 3 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", + "11987976699 1 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", + "11987984189 5239330 POD276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.a2su9qja883gs.1965276335.sqlmonitor.elapsed_time", + "11987984189 5239330 POD276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.a2su9qja883gs.1965276335.sqlmonitor.elapsed_time"); + FileUtils.writeLines(staging, lines); + SortedStagingFile sortedStagingFile = new SortedStagingFile(staging, metricIndex); + IntervalProcessors intervalProcessors = new IntervalProcessors(metricRegistry,30000, 1000000, 4, new IntervalProcessorTaskFactoryImpl(dataPointStore), 100); + IntervalProcessors.Stats stats = intervalProcessors.processFile(sortedStagingFile); + assertEquals(5, stats.nLines); + assertEquals(2, stats.nRecords); + metricIndex.close(); + } } diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestSortedStagingFile.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestSortedStagingFile.java new file mode 100644 index 00000000..8ae9a7f8 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestSortedStagingFile.java @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import com.demandware.carbonj.service.db.index.IndexUtils; +import com.demandware.carbonj.service.db.model.IntervalValues; +import com.demandware.carbonj.service.db.model.Metric; +import com.demandware.carbonj.service.db.model.MetricIndex; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class TestSortedStagingFile { + @Test + public void test() throws Exception { + MetricIndex metricIndex = IndexUtils.metricIndex(new File("/tmp/testdb"), true); + metricIndex.open(); + Metric metric1 = metricIndex.createLeafMetric("pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io"); + assertEquals(2, metric1.id); + File staging = new File("/tmp/5m7d-1734989700-9.1.s"); + List lines = List.of( + "11987976699 2 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", + "11987976699 3 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", + "11987976699 1 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", + "11987984189 5239330 POD276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.a2su9qja883gs.1965276335.sqlmonitor.elapsed_time", + "11987984189 5239330 POD276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.a2su9qja883gs.1965276335.sqlmonitor.elapsed_time"); + FileUtils.writeLines(staging, lines); + SortedStagingFile sortedStagingFile = new SortedStagingFile(staging, metricIndex); + sortedStagingFile.open(); + assertEquals("5m7d", sortedStagingFile.dbName()); + assertFalse(sortedStagingFile.isClosed()); + + IntervalValues intervalValues = sortedStagingFile.loadNeighbours().get(); + assertEquals("5m7d", intervalValues.dbName); + assertEquals(1734989700, intervalValues.intervalStart); + assertEquals(metric1, intervalValues.metric); + assertEquals(3, intervalValues.values.size()); + assertEquals(2, intervalValues.values.get(0)); + assertEquals(3, intervalValues.values.get(1)); + assertEquals(1, intervalValues.values.get(2)); + + sortedStagingFile.close(); + metricIndex.close(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestStagingFile.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestStagingFile.java new file mode 100644 index 00000000..a2528e53 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestStagingFile.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.db.index.IndexUtils; +import org.junit.jupiter.api.Test; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestStagingFile { + @Test + public void test() { + MetricRegistry metricRegistry = new MetricRegistry(); + File file = new File("/tmp/staging"); + StagingFile stagingFile = new StagingFile(metricRegistry, file, new SystemSort(), IndexUtils.metricIndex(new File("/tmp/testdb"), true)); + assertFalse(stagingFile.write(null)); + assertEquals("StagingFile{file=/tmp/staging}", stagingFile.toString()); + assertNotEquals(0, stagingFile.hashCode()); + assertTrue(stagingFile.equals(stagingFile)); + assertFalse(stagingFile.equals(new Object())); + assertFalse(stagingFile.equals(new StagingFile(metricRegistry, new File("/tmp/staging2"), new SystemSort(), IndexUtils.metricIndex(new File("/tmp/testdb"), true)))); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestStagingFileRecord.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestStagingFileRecord.java new file mode 100644 index 00000000..9722528a --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestStagingFileRecord.java @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import org.junit.jupiter.api.Test; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStagingFileRecord { + @Test + public void test() { + StagingFileSet stagingFileSet = new StagingFileSet(new File("/tmp/5m7d-1734989700-9.1.s")); + String line = "11987976699 2 pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io"; + StagingFileRecord stagingFileRecord = new StagingFileRecord(stagingFileSet, line); + assertEquals(2, stagingFileRecord.val()); + assertEquals(11987976699L, stagingFileRecord.metricId); + assertEquals("pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io", stagingFileRecord.metricName); + assertEquals("StagingFileRecord{fileName=StagingFileSet{id='5m7d-1734989700-9'}, metricId=11987976699, strValue='2', metricName=pod276.oracle_db_server.database.crs07.blade5-6.bkvk.bkvk2.OPS_USER.6y9rrcvjksgum.851311265.io}", stagingFileRecord.toString()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestSystemSort.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestSystemSort.java new file mode 100644 index 00000000..fd669d61 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/points/TestSystemSort.java @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.points; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestSystemSort { + @Test + public void testSystemSort() throws Exception { + SystemSort systemSort = new SystemSort(); + String metric1 = "5980273329 6.29 pod279.gm.prd.util1.api.ServiceHelper.isCloudV1Instance.min"; + String metric2 = "5978422979 228.19 pod279.gm.prd.util1.api.ServiceHelper.isInAdminMode.max"; + File stage1 = new File("/tmp/test_stage1"); + FileUtils.writeLines(stage1, List.of(metric1)); + File stage2 = new File("/tmp/test_stage2"); + FileUtils.writeLines(stage2, List.of(metric2)); + File stageSorted = new File("/tmp/test_stage_sorted"); + systemSort.sort(stage1, Optional.of(stage2), stageSorted); + List lines = FileUtils.readLines(stageSorted, StandardCharsets.UTF_8); + assertEquals(metric2, lines.get(0)); + assertEquals(metric1, lines.get(1)); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestDatabaseMetrics.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestDatabaseMetrics.java new file mode 100644 index 00000000..5757541c --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestDatabaseMetrics.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.util; + +import com.codahale.metrics.MetricRegistry; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestDatabaseMetrics { + @Test + public void testDatabaseMetrics() { + MetricRegistry metricRegistry = new MetricRegistry(); + DatabaseMetrics databaseMetrics = new DatabaseMetrics(metricRegistry); + databaseMetrics.markError(); + assertEquals(1, metricRegistry.getMeters().get("db.errors").getCount()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestFileUtils.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestFileUtils.java new file mode 100644 index 00000000..ae744b12 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestFileUtils.java @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.util; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.ArrayDeque; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestFileUtils { + @Test + public void testNegatives() throws Exception { + FileUtils.dumpQueueToFile(new ArrayDeque<>(), new File("/tmp/test")); + Set lines = new HashSet<>(); + FileUtils.dumpSetToFile(lines, new File("/tmp/test")); + lines.add("test line"); + FileUtils.dumpSetToFile(lines, new File("/cannot_create/dump.txt")); + assertFalse(new File("/cannot_create/dump.txt").exists()); + FileUtils.dumpSetToFile(lines, new File("/dump.txt")); + assertFalse(new File("/dump.txt").exists()); + assertTrue(FileUtils.readFilesToSet(new File("/does_not_exist"), "prefix", false).isEmpty()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestQuota.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestQuota.java new file mode 100644 index 00000000..d179fec0 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestQuota.java @@ -0,0 +1,19 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.util; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class TestQuota { + @Test + public void test() { + Quota quota = new Quota(0, 0); + assertFalse(quota.allow()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestSystemTime.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestSystemTime.java new file mode 100644 index 00000000..4b3432f8 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/db/util/TestSystemTime.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.db.util; + +import org.junit.jupiter.api.Test; + +import java.time.Clock; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestSystemTime { + @Test + public void testSystemTime() { + Clock clock = Clock.systemUTC(); + SystemTime.setClockForTest(clock); + assertEquals(clock.millis() / 1000, SystemTime.nowEpochSecond()); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/PointProcessorMock.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/PointProcessorMock.java new file mode 100644 index 00000000..d5960691 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/PointProcessorMock.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.engine; + +import com.demandware.carbonj.service.accumulator.Accumulator; + +import java.util.List; + +public class PointProcessorMock implements PointProcessor { + @Override + public void process(List points) { + + } + + @Override + public void drain() { + + } + + @Override + public void close() { + + } + + @Override + public Accumulator getAccumulator() { + return null; + } + + @Override + public void flushAggregations(boolean force) { + + } + + @Override + public void dumpStats() { + + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/RelayRulesTest.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/RelayRulesTest.java index 0671dc7d..5cc164dc 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/RelayRulesTest.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/RelayRulesTest.java @@ -10,11 +10,15 @@ import com.demandware.carbonj.service.strings.StringsCache; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.web.client.RestTemplate; import java.io.File; import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class RelayRulesTest { @@ -42,5 +46,40 @@ public void testRelayRules() { assertEquals(2, destinationGroups.length); assertEquals("kinesis:umon-prd-v2-cjajna", destinationGroups[0]); assertEquals("kinesis:umon-prd-v2-cjArgus", destinationGroups[1]); + assertEquals(2, relayRules.allDestinationGroups().size()); + relayRules.getDestinationGroups("pod240.ecom.host.jvm.memory.heap.usage"); + } + + @Test + public void testEmptyRelayRules() { + RelayRules relayRules = new RelayRules("relay"); + assertEquals(0, relayRules.allDestinationGroups().size()); + assertEquals(0, relayRules.getDestinationGroups("foo.bar").length); + assertTrue(relayRules.equals(relayRules)); + assertFalse(relayRules.equals(null)); + assertEquals(relayRules.hashCode(), new RelayRules("relay").hashCode()); + assertEquals("RelayRules{configLines=[]}", relayRules.toString()); + } + + @Test + public void testNegatives() { + RelayRules relayRules = new RelayRules("relay", + new File("/tmp/relay-rules.conf.not.exist"), + "server", true, null); + assertEquals(0, relayRules.allDestinationGroups().size()); + try { + new RelayRules("relay", + new File("/tmp/relay-rules.conf.not.exist"), + "invalid", true, null); + fail("Should have thrown exception"); + } catch (IllegalStateException e) { + assertEquals("Unexpected configuration src: invalid", e.getMessage()); + } + } + + @Test + public void testConfigServerRelayRules() throws Exception { + ConfigServerUtil configServerUtil = new ConfigServerUtil(new RestTemplate(), "http://127.0.0.1:8888", new MetricRegistry(), + "testConfigServerRelayRules", "/tmp/backup"); } } diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestInputQueue.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestInputQueue.java new file mode 100644 index 00000000..1cfb2b3b --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestInputQueue.java @@ -0,0 +1,21 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.engine; + +import com.codahale.metrics.MetricRegistry; +import org.junit.jupiter.api.Test; + +public class TestInputQueue { + @Test + public void testRejectPolicy() { + // TODO: This is just for dummy code coverage, we should really test the reject policy + InputQueue inputQueue = new InputQueue(new MetricRegistry(), "input-queue-consumer-1", new PointProcessorMock(), 1, + "block", 10, 100); + inputQueue = new InputQueue(new MetricRegistry(), "input-queue-consumer-2", new PointProcessorMock(), 1, + "drop", 10, 100); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestFileDestination.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestFileDestination.java index 3591c12d..0433d7c6 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestFileDestination.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestFileDestination.java @@ -42,6 +42,15 @@ public void test() throws Exception { checkMeter(metricRegistry, fileDestination.name + ".sent", 1L); } + @Test + public void testNegatives() throws Exception { + MetricRegistry metricRegistry = new MetricRegistry(); + FileDestination fileDestination = new FileDestination(metricRegistry, "audit", "/invalid_dir/file_destination_test", 1); + Thread.sleep(500); + fileDestination.closeQuietly(); + } + + private void checkMeter(MetricRegistry metricRegistry, String name, long expected) { Meter meter = metricRegistry.getMeters().get(name); assertNotNull(meter); diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestKinesisProducerTask.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestKinesisProducerTask.java new file mode 100644 index 00000000..ca74559d --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestKinesisProducerTask.java @@ -0,0 +1,367 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.engine.destination; + +import com.amazonaws.AmazonWebServiceRequest; +import com.amazonaws.ResponseMetadata; +import com.amazonaws.regions.Region; +import com.amazonaws.services.kinesis.AmazonKinesis; +import com.amazonaws.services.kinesis.model.AddTagsToStreamRequest; +import com.amazonaws.services.kinesis.model.AddTagsToStreamResult; +import com.amazonaws.services.kinesis.model.CreateStreamRequest; +import com.amazonaws.services.kinesis.model.CreateStreamResult; +import com.amazonaws.services.kinesis.model.DecreaseStreamRetentionPeriodRequest; +import com.amazonaws.services.kinesis.model.DecreaseStreamRetentionPeriodResult; +import com.amazonaws.services.kinesis.model.DeleteResourcePolicyRequest; +import com.amazonaws.services.kinesis.model.DeleteResourcePolicyResult; +import com.amazonaws.services.kinesis.model.DeleteStreamRequest; +import com.amazonaws.services.kinesis.model.DeleteStreamResult; +import com.amazonaws.services.kinesis.model.DeregisterStreamConsumerRequest; +import com.amazonaws.services.kinesis.model.DeregisterStreamConsumerResult; +import com.amazonaws.services.kinesis.model.DescribeLimitsRequest; +import com.amazonaws.services.kinesis.model.DescribeLimitsResult; +import com.amazonaws.services.kinesis.model.DescribeStreamConsumerRequest; +import com.amazonaws.services.kinesis.model.DescribeStreamConsumerResult; +import com.amazonaws.services.kinesis.model.DescribeStreamRequest; +import com.amazonaws.services.kinesis.model.DescribeStreamResult; +import com.amazonaws.services.kinesis.model.DescribeStreamSummaryRequest; +import com.amazonaws.services.kinesis.model.DescribeStreamSummaryResult; +import com.amazonaws.services.kinesis.model.DisableEnhancedMonitoringRequest; +import com.amazonaws.services.kinesis.model.DisableEnhancedMonitoringResult; +import com.amazonaws.services.kinesis.model.EnableEnhancedMonitoringRequest; +import com.amazonaws.services.kinesis.model.EnableEnhancedMonitoringResult; +import com.amazonaws.services.kinesis.model.GetRecordsRequest; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.amazonaws.services.kinesis.model.GetResourcePolicyRequest; +import com.amazonaws.services.kinesis.model.GetResourcePolicyResult; +import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; +import com.amazonaws.services.kinesis.model.GetShardIteratorResult; +import com.amazonaws.services.kinesis.model.IncreaseStreamRetentionPeriodRequest; +import com.amazonaws.services.kinesis.model.IncreaseStreamRetentionPeriodResult; +import com.amazonaws.services.kinesis.model.ListShardsRequest; +import com.amazonaws.services.kinesis.model.ListShardsResult; +import com.amazonaws.services.kinesis.model.ListStreamConsumersRequest; +import com.amazonaws.services.kinesis.model.ListStreamConsumersResult; +import com.amazonaws.services.kinesis.model.ListStreamsRequest; +import com.amazonaws.services.kinesis.model.ListStreamsResult; +import com.amazonaws.services.kinesis.model.ListTagsForStreamRequest; +import com.amazonaws.services.kinesis.model.ListTagsForStreamResult; +import com.amazonaws.services.kinesis.model.MergeShardsRequest; +import com.amazonaws.services.kinesis.model.MergeShardsResult; +import com.amazonaws.services.kinesis.model.PutRecordRequest; +import com.amazonaws.services.kinesis.model.PutRecordResult; +import com.amazonaws.services.kinesis.model.PutRecordsRequest; +import com.amazonaws.services.kinesis.model.PutRecordsResult; +import com.amazonaws.services.kinesis.model.PutResourcePolicyRequest; +import com.amazonaws.services.kinesis.model.PutResourcePolicyResult; +import com.amazonaws.services.kinesis.model.RegisterStreamConsumerRequest; +import com.amazonaws.services.kinesis.model.RegisterStreamConsumerResult; +import com.amazonaws.services.kinesis.model.RemoveTagsFromStreamRequest; +import com.amazonaws.services.kinesis.model.RemoveTagsFromStreamResult; +import com.amazonaws.services.kinesis.model.SplitShardRequest; +import com.amazonaws.services.kinesis.model.SplitShardResult; +import com.amazonaws.services.kinesis.model.StartStreamEncryptionRequest; +import com.amazonaws.services.kinesis.model.StartStreamEncryptionResult; +import com.amazonaws.services.kinesis.model.StopStreamEncryptionRequest; +import com.amazonaws.services.kinesis.model.StopStreamEncryptionResult; +import com.amazonaws.services.kinesis.model.UpdateShardCountRequest; +import com.amazonaws.services.kinesis.model.UpdateShardCountResult; +import com.amazonaws.services.kinesis.model.UpdateStreamModeRequest; +import com.amazonaws.services.kinesis.model.UpdateStreamModeResult; +import com.amazonaws.services.kinesis.waiters.AmazonKinesisWaiters; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.engine.DataPoint; +import com.demandware.carbonj.service.engine.kinesis.GzipDataPointCodec; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestKinesisProducerTask { + @Test + public void test() { + MetricRegistry metricRegistry = new MetricRegistry(); + Meter sent = metricRegistry.meter(MetricRegistry.name("kinesis", "sent")); + Meter drop = metricRegistry.meter(MetricRegistry.name("kinesis", "drop")); + Meter messagesSent = metricRegistry.meter(MetricRegistry.name("kinesis", "messagesSent")); + Histogram messageSize = metricRegistry.histogram(MetricRegistry.name("kinesis", "messageSize")); + Histogram dataPointsPerMessage = metricRegistry.histogram(MetricRegistry.name("kinesis", "dataPointsPerMessage")); + KinesisProducerTask kinesisProducerTask = new KinesisProducerTask(metricRegistry, new MockAmazonKinesis(), "test-stream", + List.of(new DataPoint("foo.bar", 123, 0, false)), + sent, + drop, + messagesSent, + messageSize, + metricRegistry.meter(MetricRegistry.name("kinesis", "putRetry")), + metricRegistry.timer(MetricRegistry.name("kinesis", "producer")).time(), + dataPointsPerMessage, + new GzipDataPointCodec()); + kinesisProducerTask.run(); + assertEquals(1, messageSize.getCount()); + assertEquals(1, sent.getCount()); + assertEquals(0, drop.getCount()); + assertEquals(1, dataPointsPerMessage.getCount()); + assertEquals(1, messagesSent.getCount()); + } + + private static class MockAmazonKinesis implements AmazonKinesis { + + @Override + public void setEndpoint(String s) { + + } + + @Override + public void setRegion(Region region) { + + } + + @Override + public AddTagsToStreamResult addTagsToStream(AddTagsToStreamRequest addTagsToStreamRequest) { + return null; + } + + @Override + public CreateStreamResult createStream(CreateStreamRequest createStreamRequest) { + return null; + } + + @Override + public CreateStreamResult createStream(String s, Integer integer) { + return null; + } + + @Override + public DecreaseStreamRetentionPeriodResult decreaseStreamRetentionPeriod(DecreaseStreamRetentionPeriodRequest decreaseStreamRetentionPeriodRequest) { + return null; + } + + @Override + public DeleteResourcePolicyResult deleteResourcePolicy(DeleteResourcePolicyRequest deleteResourcePolicyRequest) { + return null; + } + + @Override + public DeleteStreamResult deleteStream(DeleteStreamRequest deleteStreamRequest) { + return null; + } + + @Override + public DeleteStreamResult deleteStream(String s) { + return null; + } + + @Override + public DeregisterStreamConsumerResult deregisterStreamConsumer(DeregisterStreamConsumerRequest deregisterStreamConsumerRequest) { + return null; + } + + @Override + public DescribeLimitsResult describeLimits(DescribeLimitsRequest describeLimitsRequest) { + return null; + } + + @Override + public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) { + return null; + } + + @Override + public DescribeStreamResult describeStream(String s) { + return null; + } + + @Override + public DescribeStreamResult describeStream(String s, String s1) { + return null; + } + + @Override + public DescribeStreamResult describeStream(String s, Integer integer, String s1) { + return null; + } + + @Override + public DescribeStreamConsumerResult describeStreamConsumer(DescribeStreamConsumerRequest describeStreamConsumerRequest) { + return null; + } + + @Override + public DescribeStreamSummaryResult describeStreamSummary(DescribeStreamSummaryRequest describeStreamSummaryRequest) { + return null; + } + + @Override + public DisableEnhancedMonitoringResult disableEnhancedMonitoring(DisableEnhancedMonitoringRequest disableEnhancedMonitoringRequest) { + return null; + } + + @Override + public EnableEnhancedMonitoringResult enableEnhancedMonitoring(EnableEnhancedMonitoringRequest enableEnhancedMonitoringRequest) { + return null; + } + + @Override + public GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest) { + return null; + } + + @Override + public GetResourcePolicyResult getResourcePolicy(GetResourcePolicyRequest getResourcePolicyRequest) { + return null; + } + + @Override + public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest) { + return null; + } + + @Override + public GetShardIteratorResult getShardIterator(String s, String s1, String s2) { + return null; + } + + @Override + public GetShardIteratorResult getShardIterator(String s, String s1, String s2, String s3) { + return null; + } + + @Override + public IncreaseStreamRetentionPeriodResult increaseStreamRetentionPeriod(IncreaseStreamRetentionPeriodRequest increaseStreamRetentionPeriodRequest) { + return null; + } + + @Override + public ListShardsResult listShards(ListShardsRequest listShardsRequest) { + return null; + } + + @Override + public ListStreamConsumersResult listStreamConsumers(ListStreamConsumersRequest listStreamConsumersRequest) { + return null; + } + + @Override + public ListStreamsResult listStreams(ListStreamsRequest listStreamsRequest) { + return null; + } + + @Override + public ListStreamsResult listStreams() { + return null; + } + + @Override + public ListStreamsResult listStreams(String s) { + return null; + } + + @Override + public ListStreamsResult listStreams(Integer integer, String s) { + return null; + } + + @Override + public ListTagsForStreamResult listTagsForStream(ListTagsForStreamRequest listTagsForStreamRequest) { + return null; + } + + @Override + public MergeShardsResult mergeShards(MergeShardsRequest mergeShardsRequest) { + return null; + } + + @Override + public MergeShardsResult mergeShards(String s, String s1, String s2) { + return null; + } + + @Override + public PutRecordResult putRecord(PutRecordRequest putRecordRequest) { + return new PutRecordResult(); + } + + @Override + public PutRecordResult putRecord(String s, ByteBuffer byteBuffer, String s1) { + return null; + } + + @Override + public PutRecordResult putRecord(String s, ByteBuffer byteBuffer, String s1, String s2) { + return null; + } + + @Override + public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) { + return null; + } + + @Override + public PutResourcePolicyResult putResourcePolicy(PutResourcePolicyRequest putResourcePolicyRequest) { + return null; + } + + @Override + public RegisterStreamConsumerResult registerStreamConsumer(RegisterStreamConsumerRequest registerStreamConsumerRequest) { + return null; + } + + @Override + public RemoveTagsFromStreamResult removeTagsFromStream(RemoveTagsFromStreamRequest removeTagsFromStreamRequest) { + return null; + } + + @Override + public SplitShardResult splitShard(SplitShardRequest splitShardRequest) { + return null; + } + + @Override + public SplitShardResult splitShard(String s, String s1, String s2) { + return null; + } + + @Override + public StartStreamEncryptionResult startStreamEncryption(StartStreamEncryptionRequest startStreamEncryptionRequest) { + return null; + } + + @Override + public StopStreamEncryptionResult stopStreamEncryption(StopStreamEncryptionRequest stopStreamEncryptionRequest) { + return null; + } + + @Override + public UpdateShardCountResult updateShardCount(UpdateShardCountRequest updateShardCountRequest) { + return null; + } + + @Override + public UpdateStreamModeResult updateStreamMode(UpdateStreamModeRequest updateStreamModeRequest) { + return null; + } + + @Override + public void shutdown() { + + } + + @Override + public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest amazonWebServiceRequest) { + return null; + } + + @Override + public AmazonKinesisWaiters waiters() { + return null; + } + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestLineProtocolDestinationSocket.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestLineProtocolDestinationSocket.java index 80b48af0..56751bab 100644 --- a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestLineProtocolDestinationSocket.java +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/destination/TestLineProtocolDestinationSocket.java @@ -57,6 +57,16 @@ public void test() throws Exception { checkMeter(metricRegistry, lineProtocolDestinationSocket.name + ".sent", 1L); } + @Test + public void testNegatives() throws Exception { + DataPoint dataPoint = new DataPoint("foo.bar", 123, (int) (System.currentTimeMillis() / 1000)); + MetricRegistry metricRegistry = new MetricRegistry(); + LineProtocolDestinationSocket lineProtocolDestinationSocket = + new LineProtocolDestinationSocket(metricRegistry, "relay", "127.0.0.1", 777, 1, 1); + Thread.sleep(500); + lineProtocolDestinationSocket.closeQuietly(); + } + private void checkMeter(MetricRegistry metricRegistry, String name, long expected) { Meter meter = metricRegistry.getMeters().get(name); assertNotNull(meter); diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/kinesis/TestGzipDataPointCodec.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/kinesis/TestGzipDataPointCodec.java new file mode 100644 index 00000000..d73db82a --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/engine/kinesis/TestGzipDataPointCodec.java @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.engine.kinesis; + +import com.demandware.carbonj.service.engine.DataPoint; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestGzipDataPointCodec { + @Test + public void testDefault() { + int current = (int) (System.currentTimeMillis() / 1000); + DataPoint dataPoint = new DataPoint("foo.bar", 123, current); + List dataPoints = new ArrayList<>(); + dataPoints.add(dataPoint); + DataPoints codecDataPoints = new DataPoints(dataPoints, current); + GzipDataPointCodec codec = new GzipDataPointCodec(); + byte[] encoded = codec.encode(codecDataPoints); + DataPoints decodedDataPoints = codec.decode(encoded); + assertEquals(codecDataPoints.getTimeStamp(), decodedDataPoints.getTimeStamp()); + assertEquals(codecDataPoints.getDataPoints().size(), decodedDataPoints.getDataPoints().size()); + assertEquals(codecDataPoints.getDataPoints().get(0), decodedDataPoints.getDataPoints().get(0)); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestCarbonjEventsQueueProcessor.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestCarbonjEventsQueueProcessor.java new file mode 100644 index 00000000..5ce31281 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestCarbonjEventsQueueProcessor.java @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.events; + +import com.codahale.metrics.MetricRegistry; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.LinkedList; + +public class TestCarbonjEventsQueueProcessor { + @Test + public void test() { + CarbonjEventsQueueProcessor carbonjEventsQueueProcessor = new CarbonjEventsQueueProcessor(new MetricRegistry(), new NoOpLogger<>()); + Collection jsonObjectCollection = new LinkedList<>(); + jsonObjectCollection.add(new JsonObject()); + carbonjEventsQueueProcessor.process(jsonObjectCollection); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestEventsHandlerImpl.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestEventsHandlerImpl.java new file mode 100644 index 00000000..1c5071a3 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestEventsHandlerImpl.java @@ -0,0 +1,20 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.events; + +import io.netty.buffer.Unpooled; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +public class TestEventsHandlerImpl { + @Test + public void test() { + EventsHandlerImpl eventsHandler = new EventsHandlerImpl(new NoOpLogger<>()); + eventsHandler.process(Unpooled.wrappedBuffer("test".getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestKinesisQueueProcessor.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestKinesisQueueProcessor.java new file mode 100644 index 00000000..9595f3bc --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/events/TestKinesisQueueProcessor.java @@ -0,0 +1,19 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.events; + +import com.codahale.metrics.MetricRegistry; +import org.junit.jupiter.api.Test; + +public class TestKinesisQueueProcessor { + @Test + public void test() { + KinesisQueueProcessor kinesisQueueProcessor = new KinesisQueueProcessor(new MetricRegistry(), "test", null, 1); + kinesisQueueProcessor.refreshStats(); + kinesisQueueProcessor.dumpStats(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/ns/TestNamespaceCounter.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/ns/TestNamespaceCounter.java new file mode 100644 index 00000000..0f1c3368 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/ns/TestNamespaceCounter.java @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.ns; + +import com.codahale.metrics.MetricRegistry; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestNamespaceCounter { + @Test + public void testDefault() { + MetricRegistry metricRegistry = new MetricRegistry(); + NamespaceCounter namespaceCounter = new NamespaceCounter(metricRegistry, 0); + namespaceCounter.count("foo.bar"); + assertTrue(namespaceCounter.exists("foo.bar")); + assertTrue(namespaceCounter.getLiveNamespaces().contains("foo")); + namespaceCounter.removeInactive(); + assertFalse(namespaceCounter.exists("foo.bar")); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/queue/TestInputQueue.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/queue/TestInputQueue.java new file mode 100644 index 00000000..60dde7e3 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/queue/TestInputQueue.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.queue; + +import com.codahale.metrics.MetricRegistry; +import com.demandware.carbonj.service.events.DropRejectionHandler; +import com.demandware.carbonj.service.events.TestCarbonjEventsLogger; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestInputQueue { + @Test + public void testDefault() { + MetricRegistry metricRegistry = new MetricRegistry(); + InputQueue inputQueue = new InputQueue(metricRegistry, "test", new TestCarbonjEventsLogger.MockQueueProcessor(), + 1, new DropRejectionHandler(), 1, 1000, 1000); + inputQueue.accept(new JsonObject()); + assertEquals(1, inputQueue.queuedItemsCount()); + inputQueue.accept(new JsonObject()); + assertEquals(1, inputQueue.queuedItemsCount()); + inputQueue.refreshStats(); + inputQueue.dumpStats(); + } +} diff --git a/carbonj.service/src/test/java/com/demandware/carbonj/service/strings/TestStringsCache.java b/carbonj.service/src/test/java/com/demandware/carbonj/service/strings/TestStringsCache.java new file mode 100644 index 00000000..a83b66f8 --- /dev/null +++ b/carbonj.service/src/test/java/com/demandware/carbonj/service/strings/TestStringsCache.java @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.demandware.carbonj.service.strings; + +import com.codahale.metrics.MetricRegistry; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestStringsCache { + @Test + public void testDefault() { + MetricRegistry metricRegistry = new MetricRegistry(); + new StringsCache(metricRegistry, 0, 0, 0, 0); + assertNull(StringsCache.getState("foo.bar")); + new StringsCache(metricRegistry, 1, 1, 1, 1); + StringsCache.State state = StringsCache.getState("foo.bar"); + assertNotNull(state); + assertEquals("foo.bar", state.getKey()); + assertNull(state.getBlackListed()); + assertNull(state.getRelayDestinations()); + assertTrue(state.getAggregationRuleMap().isEmpty()); + } +} diff --git a/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/TestMessage.java b/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/TestMessage.java new file mode 100644 index 00000000..b1e423dd --- /dev/null +++ b/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/TestMessage.java @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.salesforce.cc.infra.core.kinesis; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestMessage { + @Test + public void testMessageBuilder() { + byte[] payload = "test".getBytes(StandardCharsets.UTF_8); + Message message = new Message.Builder(payload).addHeader("foo", "bar").build(); + assertEquals("bar", message.getHeader("foo")); + assertEquals("test", message.getHeader("invalid", "test")); + assertEquals(payload, message.getPayload()); + } +} diff --git a/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/parser/TestDeprecatedParser.java b/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/parser/TestDeprecatedParser.java new file mode 100644 index 00000000..6d33d4e8 --- /dev/null +++ b/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/parser/TestDeprecatedParser.java @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.salesforce.cc.infra.core.kinesis.parser; + +import com.salesforce.cc.infra.core.kinesis.Message; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestDeprecatedParser { + @Test + public void testDefault() { + DeprecatedParser parser = new DeprecatedParser(); + try { + parser.encode(null); + fail("Should have thrown an exception"); + } catch (UnsupportedOperationException e) { + assertEquals("Deprecated", e.getMessage()); + } + byte[] bytes = "test".getBytes(StandardCharsets.UTF_8); + Message message = parser.decode(bytes); + assertTrue(message.getHeaders().isEmpty()); + assertEquals(bytes, message.getPayload()); + } +} diff --git a/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/parser/TestParserFactory.java b/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/parser/TestParserFactory.java new file mode 100644 index 00000000..e25633b6 --- /dev/null +++ b/carbonj.service/src/test/java/com/salesforce/cc/infra/core/kinesis/parser/TestParserFactory.java @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2018, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +package com.salesforce.cc.infra.core.kinesis.parser; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestParserFactory { + @Test + public void testDefault() throws Exception { + assertInstanceOf(DeprecatedParser.class, ParserFactory.getParser("0")); + try { + ParserFactory.getParser("2.0"); + fail("Should have thrown an exception"); + } catch (RuntimeException e) { + assertEquals("Unsupported version for parser: 2.0", e.getMessage()); + } + byte[] bytes = "UMONTP/2.0".getBytes(StandardCharsets.UTF_8); + try { + ParserFactory.getParser(bytes); + fail("Should have thrown an exception"); + } catch (RuntimeException e) { + assertEquals("Unsupported versionLine for protocol", e.getMessage()); + } + bytes = "INVALID/1.0".getBytes(StandardCharsets.UTF_8); + assertInstanceOf(DeprecatedParser.class, ParserFactory.getParser(bytes)); + } +} diff --git a/carbonj.service/src/test/resources/logback-spring.xml b/carbonj.service/src/test/resources/logback-spring.xml new file mode 100644 index 00000000..39188813 --- /dev/null +++ b/carbonj.service/src/test/resources/logback-spring.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + ${FILE_LOG_PATTERN} + + ${LOG_FILE} + + ${LOG_FILE}.%d{yyyy-MM-dd}.log + 5 + + + + + ${FILE_LOG_PATTERN} + + ${KINESIS_LOG_FILE} + + ${KINESIS_LOG_FILE}.%d{yyyy-MM-dd}.log + 5 + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/carbonj.service/src/test/resources/storage-aggregation.conf b/carbonj.service/src/test/resources/storage-aggregation.conf index d54ea1d2..38a418b4 100644 --- a/carbonj.service/src/test/resources/storage-aggregation.conf +++ b/carbonj.service/src/test/resources/storage-aggregation.conf @@ -1,2 +1,4 @@ sum = \.count$ +min = .*\Q.min\E$ +max = .*\Q.max\E$ avg = *