From 6c58da1479276d4f6bb44712a6664dbc574be39d Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Fri, 11 Aug 2023 11:37:13 -0300 Subject: [PATCH 1/8] Write VAA count to an infinite retention bucket --- analytics/metric/metric.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/metric/metric.go b/analytics/metric/metric.go index 8d81401cd..458ee39dc 100644 --- a/analytics/metric/metric.go +++ b/analytics/metric/metric.go @@ -133,7 +133,7 @@ func (m *Metric) vaaCountMeasurement(ctx context.Context, vaa *sdk.VAA) error { } // Write the point to influx - err = m.apiBucket30Days.WritePoint(ctx, point) + err = m.apiBucketInfinite.WritePoint(ctx, point) if err != nil { m.logger.Error("failed to write metric", zap.String("measurement", point.Name()), From 3a7b128c5bd27738cbbda3389d1fafef447efb1e Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Fri, 11 Aug 2023 11:42:19 -0300 Subject: [PATCH 2/8] Remove deprecated InfluxDB task --- analytics/scripts/vaa_count_1h.flux | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 analytics/scripts/vaa_count_1h.flux diff --git a/analytics/scripts/vaa_count_1h.flux b/analytics/scripts/vaa_count_1h.flux deleted file mode 100644 index e076d245a..000000000 --- a/analytics/scripts/vaa_count_1h.flux +++ /dev/null @@ -1,17 +0,0 @@ -import "date" - -option task = { - name: "vaa_count grouped by hour", - every: 1h, -} - -start = date.truncate(t: -1h, unit: 1h) -stop = date.truncate(t: now(), unit: 1h) - -from(bucket: "wormscan-30days") - |> range(start: start, stop: stop) - |> filter(fn: (r) => r["_measurement"] == "vaa_count") - |> group() - |> aggregateWindow(every: 1h, fn: count, createEmpty: true) - |> set(key: "_measurement", value: "vaa_count_1h") - |> to(bucket: "wormscan-30days", fieldFn: (r) => ({"count": r._value})) From a583d81dbc405f6018e7c28728deefd9a866c2a3 Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Mon, 14 Aug 2023 15:43:42 -0300 Subject: [PATCH 3/8] Write vaa count to 30-day retention bucket --- analytics/metric/metric.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/analytics/metric/metric.go b/analytics/metric/metric.go index 458ee39dc..b2fc99d98 100644 --- a/analytics/metric/metric.go +++ b/analytics/metric/metric.go @@ -133,7 +133,7 @@ func (m *Metric) vaaCountMeasurement(ctx context.Context, vaa *sdk.VAA) error { } // Write the point to influx - err = m.apiBucketInfinite.WritePoint(ctx, point) + err = m.apiBucket30Days.WritePoint(ctx, point) if err != nil { m.logger.Error("failed to write metric", zap.String("measurement", point.Name()), @@ -145,6 +145,8 @@ func (m *Metric) vaaCountMeasurement(ctx context.Context, vaa *sdk.VAA) error { } m.metrics.IncSuccessfulMeasurement(vaaCountMeasurement) + m.logger.Info("generated a data point for the vaa_count metric") + return nil } From ced17ebfae9caf4fec0d68ec38265457abd47373 Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Mon, 14 Aug 2023 16:40:59 -0300 Subject: [PATCH 4/8] Add 3-month filter to last-txs endpoint --- analytics/metric/metric.go | 2 +- api/handlers/transactions/queries.go | 15 ++++++++++++--- api/handlers/transactions/queries_test.go | 8 ++++---- api/handlers/transactions/repository.go | 6 ++++-- api/middleware/extract_parameters.go | 6 ++++++ api/routes/wormscan/transactions/controller.go | 1 + 6 files changed, 28 insertions(+), 10 deletions(-) diff --git a/analytics/metric/metric.go b/analytics/metric/metric.go index b2fc99d98..df71acb71 100644 --- a/analytics/metric/metric.go +++ b/analytics/metric/metric.go @@ -145,7 +145,7 @@ func (m *Metric) vaaCountMeasurement(ctx context.Context, vaa *sdk.VAA) error { } m.metrics.IncSuccessfulMeasurement(vaaCountMeasurement) - m.logger.Info("generated a data point for the vaa_count metric") + m.logger.Info("generated a data point for the vaa count metric") return nil } diff --git a/api/handlers/transactions/queries.go b/api/handlers/transactions/queries.go index bbe791886..b2e4d3b09 100644 --- a/api/handlers/transactions/queries.go +++ b/api/handlers/transactions/queries.go @@ -36,12 +36,18 @@ union(tables: [aggregatesVaaCount, lastVaaCount]) |> sort(columns: ["_time"], desc: true) ` -func buildLastTrxQuery(bucket string, tm time.Time, q *TransactionCountQuery) string { +func buildLastTrxQuery( + dataPointsBucket string, + aggregationsBucket string, + tm time.Time, + q *TransactionCountQuery, +) string { + startLastVaa, startAggregatesVaa := createRangeQuery(tm, q.TimeSpan) if q.TimeSpan == "1d" && q.SampleRate == "1h" { - return fmt.Sprintf(queryTemplateVaaCount1d1h, bucket, startLastVaa, q.SampleRate, bucket, startAggregatesVaa) + return fmt.Sprintf(queryTemplateVaaCount1d1h, dataPointsBucket, startLastVaa, q.SampleRate, aggregationsBucket, startAggregatesVaa) } - return fmt.Sprintf(queryTemplateVaaCount, bucket, startLastVaa, bucket, startAggregatesVaa) + return fmt.Sprintf(queryTemplateVaaCount, dataPointsBucket, startLastVaa, aggregationsBucket, startAggregatesVaa) } func createRangeQuery(t time.Time, timeSpan string) (string, string) { @@ -57,6 +63,9 @@ func createRangeQuery(t time.Time, timeSpan string) (string, string) { case "1mo": startLastVaa = t.Truncate(time.Hour * 24) startAggregatesVaa = startLastVaa.Add(-time.Hour * 24 * 30) + case "3mo": + startLastVaa = t.Truncate(time.Hour * 24) + startAggregatesVaa = startLastVaa.Add(-time.Hour * 24 * 90) default: startLastVaa = t.Truncate(time.Hour * 1) startAggregatesVaa = startLastVaa.Add(-time.Hour * 24) diff --git a/api/handlers/transactions/queries_test.go b/api/handlers/transactions/queries_test.go index c337c52f8..877e5b6af 100644 --- a/api/handlers/transactions/queries_test.go +++ b/api/handlers/transactions/queries_test.go @@ -51,7 +51,7 @@ lastVaaCount = from(bucket: "wormscan-1month") |> filter(fn: (r) => r["_measurement"] == "vaa_count") |> group() |> aggregateWindow(every: 1h, fn: count, createEmpty: true) -aggregatesVaaCount = from(bucket: "wormscan-1month") +aggregatesVaaCount = from(bucket: "wormscan") |> range(start: 2023-05-03T18:00:00Z) |> filter(fn: (r) => r["_measurement"] == "vaa_count_1h") union(tables: [aggregatesVaaCount, lastVaaCount]) @@ -60,7 +60,7 @@ union(tables: [aggregatesVaaCount, lastVaaCount]) ` //2023-05-04T18:39:10.985Z tm := time.Date(2023, 5, 4, 18, 39, 10, 985, time.UTC) - actual := buildLastTrxQuery("wormscan-1month", tm, &TransactionCountQuery{TimeSpan: "1d", SampleRate: "1h"}) + actual := buildLastTrxQuery("wormscan-1month", "wormscan", tm, &TransactionCountQuery{TimeSpan: "1d", SampleRate: "1h"}) assert.Equal(t, expected, actual) } @@ -71,7 +71,7 @@ lastVaaCount = from(bucket: "wormscan-1month") |> range(start: 2023-05-04T00:00:00Z) |> filter(fn: (r) => r["_measurement"] == "vaa_count") |> group() -aggregatesVaaCount = from(bucket: "wormscan-1month") +aggregatesVaaCount = from(bucket: "wormscan") |> range(start: 2023-04-27T00:00:00Z) |> filter(fn: (r) => r["_measurement"] == "vaa_count_1h") |> aggregateWindow(every: 1h, fn: sum, createEmpty: true) @@ -82,7 +82,7 @@ union(tables: [aggregatesVaaCount, lastVaaCount]) ` //2023-05-04T18:39:10.985Z tm := time.Date(2023, 5, 4, 18, 39, 10, 985, time.UTC) - actual := buildLastTrxQuery("wormscan-1month", tm, &TransactionCountQuery{TimeSpan: "1w", SampleRate: "1d"}) + actual := buildLastTrxQuery("wormscan-1month", "wormscan", tm, &TransactionCountQuery{TimeSpan: "1w", SampleRate: "1d"}) assert.Equal(t, expected, actual) } diff --git a/api/handlers/transactions/repository.go b/api/handlers/transactions/repository.go index 03d0ae4dd..a40286374 100644 --- a/api/handlers/transactions/repository.go +++ b/api/handlers/transactions/repository.go @@ -623,7 +623,8 @@ func (r *Repository) getVolume24h(ctx context.Context) (string, error) { // GetTransactionCount get the last transactions. func (r *Repository) GetTransactionCount(ctx context.Context, q *TransactionCountQuery) ([]TransactionCountResult, error) { - query := buildLastTrxQuery(r.bucket30DaysRetention, time.Now(), q) + + query := buildLastTrxQuery(r.bucket30DaysRetention, r.bucketInfiniteRetention, time.Now(), q) result, err := r.queryAPI.Query(ctx, query) if err != nil { return nil, err @@ -631,6 +632,7 @@ func (r *Repository) GetTransactionCount(ctx context.Context, q *TransactionCoun if result.Err() != nil { return nil, result.Err() } + response := []TransactionCountResult{} for result.Next() { var row TransactionCountResult @@ -644,7 +646,7 @@ func (r *Repository) GetTransactionCount(ctx context.Context, q *TransactionCoun // https://github.com/wormhole-foundation/wormhole-explorer/issues/406 for i := range response { if i > 0 { - if q.TimeSpan == "1w" || q.TimeSpan == "1mo" { + if q.TimeSpan == "1w" || q.TimeSpan == "1mo" || q.TimeSpan == "3mo" { response[i].Time = response[i].Time.AddDate(0, 0, -1) } else if q.TimeSpan == "1d" { response[i].Time = response[i].Time.Add(-1 * time.Hour) diff --git a/api/middleware/extract_parameters.go b/api/middleware/extract_parameters.go index 2754df88c..9f297972a 100644 --- a/api/middleware/extract_parameters.go +++ b/api/middleware/extract_parameters.go @@ -297,10 +297,12 @@ func isValidSampleRate(sampleRate string) bool { } func ExtractTimeSpanAndSampleRate(c *fiber.Ctx, l *zap.Logger) (string, string, error) { + timeSpan, err := ExtractTimeSpan(c, l) if err != nil { return "", "", err } + sampleRate, err := ExtractSampleRate(c, l) if err != nil { return "", "", err @@ -319,6 +321,10 @@ func ExtractTimeSpanAndSampleRate(c *fiber.Ctx, l *zap.Logger) (string, string, if sampleRate != "1d" { return "", "", response.NewInvalidQueryParamError(c, "INVALID CONFIGURATION , QUERY PARAMETERS", nil) } + case "3mo": + if sampleRate != "1d" { + return "", "", response.NewInvalidQueryParamError(c, "INVALID CONFIGURATION , QUERY PARAMETERS", nil) + } } return timeSpan, sampleRate, nil diff --git a/api/routes/wormscan/transactions/controller.go b/api/routes/wormscan/transactions/controller.go index 66e523620..7bd271ff7 100644 --- a/api/routes/wormscan/transactions/controller.go +++ b/api/routes/wormscan/transactions/controller.go @@ -38,6 +38,7 @@ func NewController(transactionsService *transactions.Service, logger *zap.Logger // @Failure 500 // @Router /api/v1/last-txs [get] func (c *Controller) GetLastTransactions(ctx *fiber.Ctx) error { + timeSpan, sampleRate, err := middleware.ExtractTimeSpanAndSampleRate(ctx, c.logger) if err != nil { return err From 97e4dec5a2e1110165e8c510565671ca3ce40fab Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Mon, 14 Aug 2023 16:49:17 -0300 Subject: [PATCH 5/8] Update InfluxDB task for tx count summarization --- analytics/scripts/vaa_count_1h.flux | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 analytics/scripts/vaa_count_1h.flux diff --git a/analytics/scripts/vaa_count_1h.flux b/analytics/scripts/vaa_count_1h.flux new file mode 100644 index 000000000..ecc472105 --- /dev/null +++ b/analytics/scripts/vaa_count_1h.flux @@ -0,0 +1,19 @@ +import "date" + +option task = { + name: "vaa_count grouped by hour", + every: 1h, +} + +start = date.truncate(t: -1h, unit: 1h) +stop = date.truncate(t: now(), unit: 1h) +sourceBucket = "wormscan-30days" +destinationBucket = "wormscan" + +from(bucket: sourceBucket) + |> range(start: start, stop: stop) + |> filter(fn: (r) => r["_measurement"] == "vaa_count") + |> group() + |> aggregateWindow(every: 1h, fn: count, createEmpty: true) + |> set(key: "_measurement", value: "vaa_count_1h") + |> to(bucket: destinationBucket, fieldFn: (r) => ({"count": r._value})) \ No newline at end of file From 3123bccb986f9d313276c35a178f496a3cd839f6 Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Mon, 14 Aug 2023 17:16:39 -0300 Subject: [PATCH 6/8] Update parameter validations --- api/middleware/extract_parameters.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/middleware/extract_parameters.go b/api/middleware/extract_parameters.go index 9f297972a..7aa41673f 100644 --- a/api/middleware/extract_parameters.go +++ b/api/middleware/extract_parameters.go @@ -278,7 +278,7 @@ func ExtractTimeSpan(c *fiber.Ctx, l *zap.Logger) (string, error) { // isValidTimeSpan check that the timeSpan is valid. func isValidTimeSpan(timeSpan string) bool { - return regexp.MustCompile(`^1d$|^1w$|^1mo$`).MatchString(timeSpan) + return regexp.MustCompile(`^1d$|^1w$|^1mo$|^3mo$`).MatchString(timeSpan) } func ExtractSampleRate(c *fiber.Ctx, l *zap.Logger) (string, error) { From f76019658993d6f158e98121159572d1669b77f1 Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Tue, 15 Aug 2023 10:08:12 -0300 Subject: [PATCH 7/8] Improve Flux query --- api/handlers/transactions/queries.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/handlers/transactions/queries.go b/api/handlers/transactions/queries.go index b2e4d3b09..c0e3cb95b 100644 --- a/api/handlers/transactions/queries.go +++ b/api/handlers/transactions/queries.go @@ -17,6 +17,7 @@ aggregatesVaaCount = from(bucket: "%s") |> filter(fn: (r) => r["_measurement"] == "vaa_count_1h") union(tables: [aggregatesVaaCount, lastVaaCount]) |> group() + |> aggregateWindow(every: 1h, fn: sum, createEmpty: true) |> sort(columns: ["_time"], desc: true) ` From d8cc47697440bef83d6b297b8a937fd934f1b078 Mon Sep 17 00:00:00 2001 From: Agustin Godnic Date: Tue, 15 Aug 2023 15:12:18 -0300 Subject: [PATCH 8/8] Update broken test --- api/handlers/transactions/queries_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/handlers/transactions/queries_test.go b/api/handlers/transactions/queries_test.go index 877e5b6af..4f54ede28 100644 --- a/api/handlers/transactions/queries_test.go +++ b/api/handlers/transactions/queries_test.go @@ -56,6 +56,7 @@ aggregatesVaaCount = from(bucket: "wormscan") |> filter(fn: (r) => r["_measurement"] == "vaa_count_1h") union(tables: [aggregatesVaaCount, lastVaaCount]) |> group() + |> aggregateWindow(every: 1h, fn: sum, createEmpty: true) |> sort(columns: ["_time"], desc: true) ` //2023-05-04T18:39:10.985Z