diff --git a/Makefile b/Makefile index 350163a..d202209 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,7 @@ fmt: ## Format and simplify the source code using `gofmt` .PHONY: generate generate: \ + axiom/query/aggregation_string.go \ axiom/querylegacy/aggregation_string.go \ axiom/querylegacy/filter_string.go \ axiom/querylegacy/kind_string.go \ diff --git a/axiom/datasets_integration_test.go b/axiom/datasets_integration_test.go index e5cf2e3..2fe673b 100644 --- a/axiom/datasets_integration_test.go +++ b/axiom/datasets_integration_test.go @@ -230,7 +230,7 @@ func (s *DatasetsTestSuite) Test() { startTime := now.Add(-time.Minute) endTime := now.Add(time.Minute) - // Run a simple APL query. + // Run a simple APL query... apl := fmt.Sprintf("['%s']", s.dataset.ID) queryResult, err := s.client.Datasets.Query(s.ctx, apl, query.SetStartTime(startTime), @@ -401,16 +401,15 @@ func (s *DatasetsTestSuite) TestCursor() { ) s.Require().NoError(err) - // FIXME(lukasmalkmus): Tabular results format is not yet returning the - // _rowID column. - s.T().Skip() - // HINT(lukasmalkmus): Expecting four columns: _time, _sysTime, _rowID, foo. // This is only checked once for the first query result to verify the // dataset scheme. The following queries will only check the results in the // columns. + // FIXME(lukasmalkmus): Tabular results format is not yet returning the + // _rowID column. s.Require().Len(queryResult.Tables, 1) - s.Require().Len(queryResult.Tables[0].Columns, 4) + s.Require().Len(queryResult.Tables[0].Columns, 3) + // s.Require().Len(queryResult.Tables[0].Columns, 4) s.Require().Len(queryResult.Tables[0].Columns[0], 3) if s.Len(queryResult.Tables, 1) { @@ -419,6 +418,10 @@ func (s *DatasetsTestSuite) TestCursor() { s.Equal("bar", queryResult.Tables[0].Columns[2][2]) } + // FIXME(lukasmalkmus): Tabular results format is not yet returning the + // _rowID column. + s.T().Skip() + // HINT(lukasmalkmus): In a real-world scenario, the cursor would be // retrieved from the query status MinCursor or MaxCursor fields, depending // on the queries sort order. diff --git a/axiom/datasets_test.go b/axiom/datasets_test.go index dd65829..287a2df 100644 --- a/axiom/datasets_test.go +++ b/axiom/datasets_test.go @@ -198,50 +198,50 @@ var ( Fields: []query.Field{ { Name: "_time", - Type: "string", + Type: query.TypeString, }, { Name: "_sysTime", - Type: "string", + Type: query.TypeString, }, { Name: "_rowId", - Type: "string", + Type: query.TypeString, }, { Name: "agent", - Type: "string", + Type: query.TypeString, }, { Name: "bytes", - Type: "float64", + Type: query.TypeReal, }, { Name: "referrer", - Type: "string", + Type: query.TypeString, }, { Name: "remote_ip", - Type: "string", + Type: query.TypeString, }, { Name: "remote_user", - Type: "string", + Type: query.TypeString, }, { Name: "request", - Type: "string", + Type: query.TypeString, }, { Name: "response", - Type: "float64", + Type: query.TypeReal, }, { Name: "time", - Type: "string", + Type: query.TypeString, }, }, - Range: &query.RangeInfo{ + Range: &query.Range{ Field: "_time", Start: parseTimeOrPanic("2023-03-21T13:38:51.735448191Z"), End: parseTimeOrPanic("2023-03-28T13:38:51.735448191Z"), @@ -1092,6 +1092,8 @@ func TestDatasetsService_Query(t *testing.T) { assert.Equal(t, expQueryRes, res) } +// TODO(lukasmalkmus): Add test for a query with an aggregation. + func TestDatasetsService_QueryLegacy(t *testing.T) { hf := func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, http.MethodPost, r.Method) diff --git a/axiom/query/aggregation.go b/axiom/query/aggregation.go new file mode 100644 index 0000000..7c3f112 --- /dev/null +++ b/axiom/query/aggregation.go @@ -0,0 +1,134 @@ +package query + +import ( + "encoding/json" + "fmt" + "strings" +) + +//go:generate go run golang.org/x/tools/cmd/stringer -type=AggregationOp -linecomment -output=aggregation_string.go + +// An AggregationOp describes the [Aggregation] operation applied on a [Field]. +type AggregationOp uint8 + +// All available [Aggregation] operations. +const ( + OpUnknown AggregationOp = iota // unknown + + OpCount // count + OpCountIf // countif + OpDistinct // distinct + OpDistinctIf // distinctif + OpSum // sum + OpSumIf // sumif + OpAvg // avg + OpAvgIf // avgif + OpMin // min + OpMinIf // minif + OpMax // max + OpMaxIf // maxif + OpTopk // topk + OpPercentiles // percentiles + OpHistogram // histogram + OpStandardDeviation // stdev + OpStandardDeviationIf // stdevif + OpVariance // variance + OpVarianceIf // varianceif + OpArgMin // argmin + OpArgMax // argmax + OpRate // rate + OpPearson // pearson_correlation + OpMakeSet // makeset + OpMakeSetIf // makesetif + OpMakeList // makelist + OpMakeListIf // makelistif +) + +func aggregationOpFromString(s string) (op AggregationOp, err error) { + switch strings.ToLower(s) { + case OpCount.String(): + op = OpCount + case OpCountIf.String(): + op = OpCountIf + case OpDistinct.String(): + op = OpDistinct + case OpDistinctIf.String(): + op = OpDistinctIf + case OpSum.String(): + op = OpSum + case OpSumIf.String(): + op = OpSumIf + case OpAvg.String(): + op = OpAvg + case OpAvgIf.String(): + op = OpAvgIf + case OpMin.String(): + op = OpMin + case OpMinIf.String(): + op = OpMinIf + case OpMax.String(): + op = OpMax + case OpMaxIf.String(): + op = OpMaxIf + case OpTopk.String(): + op = OpTopk + case OpPercentiles.String(): + op = OpPercentiles + case OpHistogram.String(): + op = OpHistogram + case OpStandardDeviation.String(): + op = OpStandardDeviation + case OpStandardDeviationIf.String(): + op = OpStandardDeviationIf + case OpVariance.String(): + op = OpVariance + case OpVarianceIf.String(): + op = OpVarianceIf + case OpArgMin.String(): + op = OpArgMin + case OpArgMax.String(): + op = OpArgMax + case OpRate.String(): + op = OpRate + case OpPearson.String(): + op = OpPearson + case OpMakeSet.String(): + op = OpMakeSet + case OpMakeSetIf.String(): + op = OpMakeSetIf + case OpMakeList.String(): + op = OpMakeList + case OpMakeListIf.String(): + op = OpMakeListIf + default: + return OpUnknown, fmt.Errorf("unknown aggregation operation: %s", s) + } + + return op, nil +} + +// UnmarshalJSON implements [json.Unmarshaler]. It is in place to unmarshal the +// AggregationOp from the string representation the server returns. +func (op *AggregationOp) UnmarshalJSON(b []byte) (err error) { + var s string + if err = json.Unmarshal(b, &s); err != nil { + return err + } + + *op, err = aggregationOpFromString(s) + + return err +} + +// Aggregation that is applied to a [Field] in a [Table]. +type Aggregation struct { + // Op is the aggregation operation. If the aggregation is aliased, the alias + // is stored in the parent [Field.Name]. + Op AggregationOp `json:"name"` + // Fields specifies the names of the fields this aggregation is computed on. + // E.g. ["players"] for "topk(players, 10)". + Fields []string `json:"fields"` + // Args are the non-field arguments of the aggregation, if any. E.g. "10" + // for "topk(players, 10)". + Args []any `json:"args"` +} diff --git a/axiom/query/aggregation_string.go b/axiom/query/aggregation_string.go new file mode 100644 index 0000000..ee4ceb7 --- /dev/null +++ b/axiom/query/aggregation_string.go @@ -0,0 +1,50 @@ +// Code generated by "stringer -type=AggregationOp -linecomment -output=aggregation_string.go"; DO NOT EDIT. + +package query + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[OpUnknown-0] + _ = x[OpCount-1] + _ = x[OpCountIf-2] + _ = x[OpDistinct-3] + _ = x[OpDistinctIf-4] + _ = x[OpSum-5] + _ = x[OpSumIf-6] + _ = x[OpAvg-7] + _ = x[OpAvgIf-8] + _ = x[OpMin-9] + _ = x[OpMinIf-10] + _ = x[OpMax-11] + _ = x[OpMaxIf-12] + _ = x[OpTopk-13] + _ = x[OpPercentiles-14] + _ = x[OpHistogram-15] + _ = x[OpStandardDeviation-16] + _ = x[OpStandardDeviationIf-17] + _ = x[OpVariance-18] + _ = x[OpVarianceIf-19] + _ = x[OpArgMin-20] + _ = x[OpArgMax-21] + _ = x[OpRate-22] + _ = x[OpPearson-23] + _ = x[OpMakeSet-24] + _ = x[OpMakeSetIf-25] + _ = x[OpMakeList-26] + _ = x[OpMakeListIf-27] +} + +const _AggregationOp_name = "unknowncountcountifdistinctdistinctifsumsumifavgavgifminminifmaxmaxiftopkpercentileshistogramstdevstdevifvariancevarianceifargminargmaxratepearson_correlationmakesetmakesetifmakelistmakelistif" + +var _AggregationOp_index = [...]uint8{0, 7, 12, 19, 27, 37, 40, 45, 48, 53, 56, 61, 64, 69, 73, 84, 93, 98, 105, 113, 123, 129, 135, 139, 158, 165, 174, 182, 192} + +func (i AggregationOp) String() string { + if i >= AggregationOp(len(_AggregationOp_index)-1) { + return "AggregationOp(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _AggregationOp_name[_AggregationOp_index[i]:_AggregationOp_index[i+1]] +} diff --git a/axiom/query/aggregation_test.go b/axiom/query/aggregation_test.go new file mode 100644 index 0000000..cd0a65e --- /dev/null +++ b/axiom/query/aggregation_test.go @@ -0,0 +1,47 @@ +package query + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAggregationOp_Unmarshal(t *testing.T) { + var act struct { + Op AggregationOp `json:"name"` + } + err := json.Unmarshal([]byte(`{ "name": "count" }`), &act) + require.NoError(t, err) + + assert.Equal(t, OpCount, act.Op) +} + +func TestAggregationOp_String(t *testing.T) { + // Check outer bounds. + assert.Equal(t, OpUnknown, AggregationOp(0)) + assert.Contains(t, (OpMakeListIf + 1).String(), "AggregationOp(") + + for op := OpUnknown; op <= OpMakeListIf; op++ { + s := op.String() + assert.NotEmpty(t, s) + assert.NotContains(t, s, "AggregationOp(") + } +} + +func TestAggregationOpFromString(t *testing.T) { + for op := OpCount; op <= OpMakeListIf; op++ { + s := op.String() + + parsedOp, err := aggregationOpFromString(s) + if assert.NoError(t, err) { + assert.NotEmpty(t, s) + assert.Equal(t, op, parsedOp) + } + } + + op, err := aggregationOpFromString("abc") + assert.Equal(t, OpUnknown, op) + assert.EqualError(t, err, "unknown aggregation operation: abc") +} diff --git a/axiom/query/field.go b/axiom/query/field.go new file mode 100644 index 0000000..997b698 --- /dev/null +++ b/axiom/query/field.go @@ -0,0 +1,126 @@ +package query + +import ( + "encoding/json" + "fmt" + "strings" +) + +// A FieldType describes the type of a [Field]. +type FieldType uint16 + +// All available [Field] types. +const ( + TypeInvalid FieldType = 0 // invalid + TypeBool FieldType = 1 << iota // bool + TypeDateTime // datetime + TypeInt // int + TypeLong // long + TypeReal // real + TypeString // string + TypeTimespan // timespan + TypeArray // array + TypeDictionary // dictionary + TypeUnknown // unknown + maxFieldType +) + +func fieldTypeFromString(s string) (ft FieldType, err error) { + types := strings.Split(s, "|") + + // FIXME(lukasmalkmus): It looks like there are more/different type aliases + // then documented: https://axiom.co/docs/apl/data-types/scalar-data-types. + for _, t := range types { + switch strings.ToLower(t) { + case TypeBool.String(), "boolean": + ft |= TypeBool + case TypeDateTime.String(), "date": + ft |= TypeDateTime + case TypeInt.String(), "integer": // "integer" is not documented. + ft |= TypeInt + case TypeLong.String(): + ft |= TypeLong + case TypeReal.String(), "double", "float", "float64": // "float" and "float64" are not documented. + ft |= TypeReal + case TypeString.String(): + ft |= TypeString + case TypeTimespan.String(), "time": + ft |= TypeTimespan + case TypeArray.String(): + ft |= TypeArray + case TypeDictionary.String(): + ft |= TypeDictionary + case TypeUnknown.String(): + ft |= TypeUnknown + default: + return TypeInvalid, fmt.Errorf("invalid field type: %s", t) + } + } + + return ft, nil +} + +// String returns a string representation of the field type. +// +// It implements [fmt.Stringer]. +func (ft FieldType) String() string { + if ft >= maxFieldType { + return fmt.Sprintf("", ft, ft) + } + + //nolint:exhaustive // maxFieldType is not a valid field type and already + // handled above. + switch ft { + case TypeBool: + return "bool" + case TypeDateTime: + return "datetime" + case TypeInt: + return "int" + case TypeLong: + return "long" + case TypeReal: + return "real" + case TypeString: + return "string" + case TypeTimespan: + return "timespan" + case TypeArray: + return "array" + case TypeDictionary: + return "dictionary" + case TypeUnknown: + return "unknown" + } + + var res []string + for fieldType := TypeBool; fieldType < maxFieldType; fieldType <<= 1 { + if ft&fieldType != 0 { + res = append(res, fieldType.String()) + } + } + return strings.Join(res, "|") +} + +// UnmarshalJSON implements [json.Unmarshaler]. It is in place to unmarshal the +// FieldType from the string representation the server returns. +func (ft *FieldType) UnmarshalJSON(b []byte) (err error) { + var s string + if err = json.Unmarshal(b, &s); err != nil { + return err + } + + *ft, err = fieldTypeFromString(s) + + return err +} + +// Field in a [Table]. +type Field struct { + // Name of the field. + Name string `json:"name"` + // Type of the field. Can also be composite types. + Type FieldType `json:"type"` + // Aggregation is the aggregation applied to the field. + Aggregation *Aggregation `json:"agg"` +} diff --git a/axiom/query/field_test.go b/axiom/query/field_test.go new file mode 100644 index 0000000..c05f8ab --- /dev/null +++ b/axiom/query/field_test.go @@ -0,0 +1,45 @@ +package query + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFieldType_Unmarshal(t *testing.T) { + var act struct { + Type FieldType `json:"type"` + } + err := json.Unmarshal([]byte(`{ "type": "int|real" }`), &act) + require.NoError(t, err) + + assert.Equal(t, TypeInt|TypeReal, act.Type) +} + +func TestFieldType_String(t *testing.T) { + assert.Equal(t, TypeInvalid, FieldType(0)) + + typ := TypeInt + assert.Equal(t, "int", typ.String()) + + typ |= TypeReal + assert.Equal(t, "int|real", typ.String()) +} + +func TestFieldTypeFromString(t *testing.T) { + for typ := TypeBool; typ <= TypeUnknown; typ <<= 1 { + s := typ.String() + + parsedOp, err := fieldTypeFromString(s) + if assert.NoError(t, err) { + assert.NotEmpty(t, s) + assert.Equal(t, typ, parsedOp) + } + } + + typ, err := fieldTypeFromString("abc") + assert.Equal(t, TypeInvalid, typ) + assert.EqualError(t, err, "invalid field type: abc") +} diff --git a/axiom/query/result.go b/axiom/query/result.go index 9cdf852..e46d531 100644 --- a/axiom/query/result.go +++ b/axiom/query/result.go @@ -36,10 +36,10 @@ type Table struct { Groups []Group `json:"groups"` // Range specifies the window the query was restricted to. Nil if the query // was not restricted to a time window. - Range *RangeInfo `json:"range"` + Range *Range `json:"range"` // Buckets defines if the query is bucketed (usually on the "_time" field). // Nil if the query returns a non-bucketed result. - Buckets *BucketInfo `json:"buckets"` + Buckets *Buckets `json:"buckets"` // Columns in the table matching the order of the [Fields] (e.g. the // [Column] at index 0 has the values for the [Field] at index 0). In case // of sub-groups, rows will repeat the group value. @@ -51,25 +51,6 @@ func (t Table) Rows() iter.Iter[Row] { return Rows(t.Columns) } -// Field in a [Table]. -type Field struct { - // Name of the field. - Name string `json:"name"` - // Type of the field. Can also be composite types which are types separated - // by a horizontal line "|". - Type string `json:"type"` - // Aggregation is the aggregation applied to the field. - Aggregation Aggregation `json:"agg"` -} - -// Aggregation that is applied to a [Field] in a [Table]. -type Aggregation struct { - // Name of the aggregation. - Name string `json:"name"` - // Args are the arguments of the aggregation. - Args []any `json:"args"` -} - // Source that was consulted in order to create a [Table]. type Source struct { // Name of the source. @@ -91,8 +72,8 @@ type Group struct { Name string `json:"name"` } -// RangeInfo specifies the window a query was restricted to. -type RangeInfo struct { +// Range specifies the window a query was restricted to. +type Range struct { // Field specifies the field name on which the query range was restricted. // Usually "_time": Field string @@ -104,9 +85,9 @@ type RangeInfo struct { End time.Time } -// BucketInfo captures information about how a grouped query is sorted into -// buckets. Usually buckets are created on the "_time" column, -type BucketInfo struct { +// Buckets captures information about how a grouped query is sorted into +// buckets. Usually buckets are created on the "_time" column. +type Buckets struct { // Field specifies the field used to create buckets on. Usually this would // be "_time". Field string