From 63dce9aa8b4a7646cd59deb5d772bfe711a7f885 Mon Sep 17 00:00:00 2001 From: makeavish Date: Thu, 13 Apr 2023 10:33:40 +0530 Subject: [PATCH 1/8] feat: traces QB checkpoint --- .../app/clickhouseReader/reader.go | 2 +- pkg/query-service/app/http_handler.go | 7 +- .../app/logs/v3/query_builder.go | 2 +- .../app/logs/v3/query_builder_test.go | 14 +- .../app/metrics/v3/query_builder.go | 2 +- .../app/metrics/v3/query_builder_test.go | 2 +- .../app/traces/v3/query_builder.go | 326 ++++++++++++++++++ .../app/traces/v3/query_builder_test.go | 137 ++++++++ pkg/query-service/constants/constants.go | 2 + pkg/query-service/model/v3/v3.go | 6 +- 10 files changed, 482 insertions(+), 18 deletions(-) create mode 100644 pkg/query-service/app/traces/v3/query_builder.go create mode 100644 pkg/query-service/app/traces/v3/query_builder_test.go diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 39c84b93b8..6622b8477f 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3827,7 +3827,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v v3.AggregateOperatorMax: where = "tagKey ILIKE $1 AND (tagDataType='int64' or tagDataType='float64')" case - v3.AggregateOpeatorCount, + v3.AggregateOperatorCount, v3.AggregateOperatorNoOp: return &v3.AggregateAttributeResponse{}, nil default: diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 9310cc5ad8..b2bce165ec 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -25,6 +25,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/metrics" metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/app/parser" + tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" @@ -107,10 +108,8 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { builderOpts := queryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, - BuildTraceQuery: func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error) { - return "", errors.New("not implemented") - }, - BuildLogQuery: logsv3.PrepareLogsQuery, + BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildLogQuery: logsv3.PrepareLogsQuery, } aH.queryBuilder = NewQueryBuilder(builderOpts) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index d2b1a145e6..35b43f9015 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -253,7 +253,7 @@ func buildLogsQuery(start, end, step int64, mq *v3.BuilderQuery, fields map[stri op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil - case v3.AggregateOpeatorCount: + case v3.AggregateOperatorCount: if mq.AggregateAttribute.Key != "" { field, err := encrichFieldWithMetadata(mq.AggregateAttribute, fields) if err != nil { diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index b67d5b4f8e..e9d984f7f1 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -58,19 +58,19 @@ var testGetSelectLabelsData = []struct { }{ { Name: "select fields for groupBy attribute", - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, SelectLabels: ", attributes_string_value[indexOf(attributes_string_key, 'user_name')] as user_name", }, { Name: "select fields for groupBy resource", - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, SelectLabels: ", resources_string_value[indexOf(resources_string_key, 'user_name')] as user_name", }, { Name: "select fields for groupBy attribute and resource", - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, GroupByTags: []v3.AttributeKey{ {Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, {Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, @@ -79,7 +79,7 @@ var testGetSelectLabelsData = []struct { }, { Name: "select fields for groupBy materialized columns", - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, GroupByTags: []v3.AttributeKey{{Key: "host", IsColumn: true}}, SelectLabels: ", host as host", }, @@ -202,7 +202,7 @@ var testBuildLogsQueryData = []struct { Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, TableName: "logs", @@ -216,7 +216,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, TableName: "logs", @@ -230,7 +230,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, - AggregateOperator: v3.AggregateOpeatorCount, + AggregateOperator: v3.AggregateOperatorCount, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: ">"}, }}, diff --git a/pkg/query-service/app/metrics/v3/query_builder.go b/pkg/query-service/app/metrics/v3/query_builder.go index 1d4eabeac4..bbba81fdd6 100644 --- a/pkg/query-service/app/metrics/v3/query_builder.go +++ b/pkg/query-service/app/metrics/v3/query_builder.go @@ -260,7 +260,7 @@ func buildMetricQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str op := fmt.Sprintf("%s(value)", aggregateOperatorToSQLFunc[mq.AggregateOperator]) query := fmt.Sprintf(queryTmpl, groupTags, step, op, filterSubQuery, groupBy, orderBy) return query, nil - case v3.AggregateOpeatorCount: + case v3.AggregateOperatorCount: op := "toFloat64(count(*))" query := fmt.Sprintf(queryTmpl, groupTags, step, op, filterSubQuery, groupBy, orderBy) return query, nil diff --git a/pkg/query-service/app/metrics/v3/query_builder_test.go b/pkg/query-service/app/metrics/v3/query_builder_test.go index bcb9cfc69e..c661c53fad 100644 --- a/pkg/query-service/app/metrics/v3/query_builder_test.go +++ b/pkg/query-service/app/metrics/v3/query_builder_test.go @@ -18,7 +18,7 @@ func TestBuildQuery(t *testing.T) { "A": { QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name"}, - AggregateOperator: v3.AggregateOperatorRateMax, + AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", }, }, diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go new file mode 100644 index 0000000000..a2579f8d4b --- /dev/null +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -0,0 +1,326 @@ +package traces + +import ( + "fmt" + "strings" + + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ + v3.AggregateOperatorP05: 0.5, + v3.AggregateOperatorP10: 0.10, + v3.AggregateOperatorP20: 0.20, + v3.AggregateOperatorP25: 0.25, + v3.AggregateOperatorP50: 0.50, + v3.AggregateOperatorP75: 0.75, + v3.AggregateOperatorP90: 0.90, + v3.AggregateOperatorP95: 0.95, + v3.AggregateOperatorP99: 0.99, + v3.AggregateOperatorHistQuant50: 0.50, + v3.AggregateOperatorHistQuant75: 0.75, + v3.AggregateOperatorHistQuant90: 0.90, + v3.AggregateOperatorHistQuant95: 0.95, + v3.AggregateOperatorHistQuant99: 0.99, +} + +var aggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{ + v3.AggregateOperatorAvg: "avg", + v3.AggregateOperatorMax: "max", + v3.AggregateOperatorMin: "min", + v3.AggregateOperatorSum: "sum", + v3.AggregateOperatorRate: "count", + v3.AggregateOperatorRateSum: "sum", + v3.AggregateOperatorRateAvg: "avg", + v3.AggregateOperatorRateMax: "max", + v3.AggregateOperatorRateMin: "min", +} + +var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ + v3.FilterOperatorIn: "IN", + v3.FilterOperatorNotIn: "NOT IN", + v3.FilterOperatorEqual: "=", + v3.FilterOperatorNotEqual: "!=", + v3.FilterOperatorLessThan: "<", + v3.FilterOperatorLessThanOrEq: "<=", + v3.FilterOperatorGreaterThan: ">", + v3.FilterOperatorGreaterThanOrEq: ">=", + v3.FilterOperatorLike: "ILIKE", + v3.FilterOperatorNotLike: "NOT ILIKE", + v3.FilterOperatorContains: "ILIKE", + v3.FilterOperatorNotContains: "NOT ILIKE", + v3.FilterOperatorExists: "IS NOT NULL", + v3.FilterOperatorNotExists: "IS NULL", +} + +func getFilter(key v3.AttributeKey) string { + if key.IsColumn { + return key.Key + } + filterType := key.Type + filterDataType := "string" + if key.DataType == v3.AttributeKeyDataTypeFloat64 || key.DataType == v3.AttributeKeyDataTypeInt64 { + filterDataType = "float64" + } + if filterType == v3.AttributeKeyTypeTag { + filterType = "TagMap" + } else { + filterType = "resourceTagsMap" + filterDataType = "" + } + return fmt.Sprintf("%s%s['%s']", filterType, filterDataType, key.Key) +} + +// getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator +func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { + var selectLabels string + if aggregatorOperator == v3.AggregateOperatorNoOp { + selectLabels = "" + } else { + for _, tag := range groupBy { + filterName := getFilter(tag) + selectLabels += fmt.Sprintf(", %s as %s", filterName, tag.Key) + } + } + return selectLabels +} + +func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { + var conditions []string + + if fs != nil && len(fs.Items) != 0 { + for _, item := range fs.Items { + toFormat := item.Value + // generate the key + columnName := getFilter(item.Key) + fmtVal := utils.ClickHouseFormattedValue(toFormat) + + if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { + conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) + } else { + return "", fmt.Errorf("unsupported operation") + } + } + } + queryString := strings.Join(conditions, " AND ") + + if len(queryString) > 0 { + queryString = " AND " + queryString + } + return queryString, nil +} + +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string) (string, error) { + + filterSubQuery, err := buildTracesFilterQuery(mq.Filters) + if err != nil { + return "", err + } + + spanIndexTableTimeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d')", start, end) + + selectLabels := getSelectLabels(mq.AggregateOperator, mq.GroupBy) + + // Select the aggregate value for interval + queryTmpl := + "SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts" + selectLabels + + ", %s as value " + + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + + " where " + spanIndexTableTimeFilter + "%s " + + "group by %s " + + "order by %sts" + + // tagsWithoutLe is used to group by all tags except le + // This is done because we want to group by le only when we are calculating quantile + // Otherwise, we want to group by all tags except le + // tagsWithoutLe := []string{} + // for _, tag := range mq.GroupBy { + // if tag.Key != "le" { + // tagsWithoutLe = append(tagsWithoutLe, tag.Key) + // } + // } + + // groupByWithoutLe := groupBy(tagsWithoutLe...) + // groupTagsWithoutLe := groupSelect(tagsWithoutLe...) + // orderWithoutLe := orderBy(mq.OrderBy, tagsWithoutLe) + + groupBy := groupByAttributeKeyTags(mq.GroupBy...) + // groupTags := groupSelectAttributeKeyTags(mq.GroupBy...) + orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) + + aggregationKey := "" + if mq.AggregateAttribute.Key != "" { + aggregationKey = getFilter(mq.AggregateAttribute) + } + + switch mq.AggregateOperator { + case v3.AggregateOperatorRateSum, + v3.AggregateOperatorRateMax, + v3.AggregateOperatorRateAvg, + v3.AggregateOperatorRateMin, + v3.AggregateOperatorRate: + rateQueryTmpl := + "SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts" + + ", %s as value " + + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + + " where " + spanIndexTableTimeFilter + "%s " + + "group by %s " + + "order by %sts" + op := fmt.Sprintf("%s(%s)/%d", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, step) + query := fmt.Sprintf(rateQueryTmpl, step, op, filterSubQuery, groupBy, orderBy) + return query, nil + case + v3.AggregateOperatorP05, + v3.AggregateOperatorP10, + v3.AggregateOperatorP20, + v3.AggregateOperatorP25, + v3.AggregateOperatorP50, + v3.AggregateOperatorP75, + v3.AggregateOperatorP90, + v3.AggregateOperatorP95, + v3.AggregateOperatorP99: + op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + return query, nil + case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: + op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + return query, nil + case v3.AggregateOperatorCount: + op := fmt.Sprintf("toFloat64(count(%s))", aggregationKey) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + return query, nil + case v3.AggregateOperatorCountDistinct: + op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + return query, nil + case v3.AggregateOperatorNoOp: + return "", fmt.Errorf("not implemented, part of traces page") + // noOpQueryTmpl := + // "SELECT fingerprint, labels as fullLabels," + + // " toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL %d SECOND) as ts," + + // " any(value) as value" + + // " FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + constants.SIGNOZ_SAMPLES_TABLENAME + + // " GLOBAL INNER JOIN" + + // " (%s)+ ORDER BY ts" + // query := fmt.Sprintf(noOpQueryTmpl, step, filterSubQuery) + // return query, nil + default: + return "", fmt.Errorf("unsupported aggregate operator") + } +} + +// groupBy returns a string of comma separated tags for group by clause +// `ts` is always added to the group by clause +func groupBy(tags ...string) string { + tags = append(tags, "ts") + return strings.Join(tags, ",") +} + +// groupSelect returns a string of comma separated tags for select clause +func groupSelect(tags ...string) string { + groupTags := strings.Join(tags, ",") + if len(tags) != 0 { + groupTags += ", " + } + return groupTags +} + +func groupByAttributeKeyTags(tags ...v3.AttributeKey) string { + groupTags := []string{} + for _, tag := range tags { + groupTags = append(groupTags, getFilter(tag)) + } + return groupBy(groupTags...) +} + +func groupSelectAttributeKeyTags(tags ...v3.AttributeKey) string { + groupTags := []string{} + for _, tag := range tags { + groupTags = append(groupTags, getFilter(tag)) + } + return groupSelect(groupTags...) +} + +// orderBy returns a string of comma separated tags for order by clause +// if the order is not specified, it defaults to ASC +func orderBy(items []v3.OrderBy, tags []string) string { + var orderBy []string + for _, tag := range tags { + found := false + for _, item := range items { + if item.ColumnName == tag { + found = true + orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) + break + } + } + if !found { + orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag)) + } + } + return strings.Join(orderBy, ",") +} + +func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string { + var groupTags []string + for _, tag := range tags { + groupTags = append(groupTags, getFilter(tag)) + } + str := orderBy(items, groupTags) + if len(str) > 0 { + str = str + "," + } + return str +} + +func having(items []v3.Having) string { + var having []string + for _, item := range items { + having = append(having, fmt.Sprintf("%s %s %v", item.ColumnName, item.Operator, utils.ClickHouseFormattedValue(item.Value))) + } + return strings.Join(having, " AND ") +} + +func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v3.AggregateOperator) (string, error) { + // var selectLabels string + var groupBy string + // NOOP can possibly return multiple time series and reduce should be applied + // for each uniques series. When the final result contains more than one series we throw + // an error post DB fetching. Otherwise just return the single data. This is not known until queried so the + // the query is prepared accordingly. + // if aggregateOperator == v3.AggregateOperatorNoOp { + // selectLabels = ", any(fullLabels) as fullLabels" + // groupBy = "GROUP BY fingerprint" + // } + // the timestamp picked is not relevant here since the final value used is show the single + // chart with just the query value. For the query + switch reduceTo { + case v3.ReduceToOperatorLast: + query = fmt.Sprintf("SELECT anyLast(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy) + case v3.ReduceToOperatorSum: + query = fmt.Sprintf("SELECT sum(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy) + case v3.ReduceToOperatorAvg: + query = fmt.Sprintf("SELECT avg(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy) + case v3.ReduceToOperatorMax: + query = fmt.Sprintf("SELECT max(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy) + case v3.ReduceToOperatorMin: + query = fmt.Sprintf("SELECT min(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy) + default: + return "", fmt.Errorf("unsupported reduce operator") + } + return query, nil +} + +func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery) (string, error) { + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME) + if err != nil { + return "", err + } + if panelType == v3.PanelTypeValue { + query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) + } + return query, err +} diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go new file mode 100644 index 0000000000..9e1b81558d --- /dev/null +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -0,0 +1,137 @@ +package traces + +import ( + "fmt" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/require" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func TestBuildQuery(t *testing.T) { + t.Run("TestSimpleQueryWithName", func(t *testing.T) { + q := &v3.QueryRangeParamsV3{ + Start: 1680066360726210000, + End: 1780066458000000000, + Step: 60, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true}, + AggregateOperator: v3.AggregateOperatorP20, + Expression: "A", + }, + }, + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeGraph, + }, + } + query, err := PrepareTracesQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"]) + require.NoError(t, err) + fmt.Println(query) + // require.Contains(t, query, "WHERE name") + }) +} + +func TestBuildQueryWithFilters(t *testing.T) { + t.Run("TestBuildQueryWithFilters", func(t *testing.T) { + q := &v3.QueryRangeParamsV3{ + Start: 1650991982000, + End: 1651078382000, + Step: 60, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "a"}, Value: "b", Operator: "neq"}, + {Key: v3.AttributeKey{Key: "code"}, Value: "ERROR_*", Operator: "nmatch"}, + }}, + AggregateOperator: v3.AggregateOperatorRateMax, + Expression: "A", + }, + }, + }, + } + query, err := PrepareTracesQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"]) + require.NoError(t, err) + + require.Contains(t, query, "WHERE metric_name = 'name' AND JSONExtractString(labels, 'a') != 'b'") + // require.Contains(t, query, rateWithoutNegative) + require.Contains(t, query, "not match(JSONExtractString(labels, 'code'), 'ERROR_*')") + }) +} + +func TestBuildQueryWithMultipleQueries(t *testing.T) { + t.Run("TestBuildQueryWithFilters", func(t *testing.T) { + q := &v3.QueryRangeParamsV3{ + Start: 1650991982000, + End: 1651078382000, + Step: 60, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: "in"}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + "B": { + QueryName: "B", + AggregateAttribute: v3.AttributeKey{Key: "name2"}, + AggregateOperator: v3.AggregateOperatorRateMax, + Expression: "B", + }, + }, + }, + } + + query, err := PrepareTracesQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"]) + require.NoError(t, err) + + require.Contains(t, query, "WHERE metric_name = 'name' AND JSONExtractString(labels, 'in') IN ['a','b','c']") + // require.Contains(t, query, rateWithoutNegative) + }) +} + +var testGetFilter = []struct { + Name string + AttributeKey v3.AttributeKey + ExpectedFilter string +}{ + { + Name: "resource", + AttributeKey: v3.AttributeKey{Key: "collector_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: false}, + ExpectedFilter: "resourcesTagMap['collector_id']", + }, + { + Name: "stringAttribute", + AttributeKey: v3.AttributeKey{Key: "customer_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedFilter: "stringTagMap['customer_id']", + }, + { + Name: "numberAttribute", + AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedFilter: "numberTagMap['count']", + }, + { + Name: "column", + AttributeKey: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + ExpectedFilter: "name", + }, +} + +func TestGetFilter(t *testing.T) { + for _, tt := range testGetFilter { + Convey("testGetFilter", t, func() { + Filter := getFilter(tt.AttributeKey) + So(Filter, ShouldEqual, tt.ExpectedFilter) + }) + } +} diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index d0ded00bbe..de2eade873 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -133,6 +133,8 @@ const ( SIGNOZ_METRIC_DBNAME = "signoz_metrics" SIGNOZ_SAMPLES_TABLENAME = "distributed_samples_v2" SIGNOZ_TIMESERIES_TABLENAME = "distributed_time_series_v2" + SIGNOZ_TRACE_DBNAME = "signoz_traces" + SIGNOZ_SPAN_INDEX_TABLENAME = "distributed_signoz_index_v2" ) var TimeoutExcludedRoutes = map[string]bool{ diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index c1dddc60f6..37880471ad 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -30,7 +30,7 @@ type AggregateOperator string const ( AggregateOperatorNoOp AggregateOperator = "noop" - AggregateOpeatorCount AggregateOperator = "count" + AggregateOperatorCount AggregateOperator = "count" AggregateOperatorCountDistinct AggregateOperator = "count_distinct" AggregateOperatorSum AggregateOperator = "sum" AggregateOperatorAvg AggregateOperator = "avg" @@ -64,7 +64,7 @@ const ( func (a AggregateOperator) Validate() error { switch a { case AggregateOperatorNoOp, - AggregateOpeatorCount, + AggregateOperatorCount, AggregateOperatorCountDistinct, AggregateOperatorSum, AggregateOperatorAvg, @@ -104,7 +104,7 @@ func (a AggregateOperator) Validate() error { func (a AggregateOperator) RequireAttribute() bool { switch a { case AggregateOperatorNoOp, - AggregateOpeatorCount, + AggregateOperatorCount, AggregateOperatorCountDistinct: return false default: From 64f8e759a0b3b0d44beb0a0e82537b81812f2d20 Mon Sep 17 00:00:00 2001 From: makeavish Date: Fri, 14 Apr 2023 13:35:34 +0530 Subject: [PATCH 2/8] feat: add support for empty tagType/dataType in attributeKey --- .../app/clickhouseReader/options.go | 3 + .../app/clickhouseReader/reader.go | 39 ++- pkg/query-service/app/http_handler.go | 24 +- pkg/query-service/app/query_builder.go | 8 +- .../app/traces/v3/query_builder.go | 234 ++++++++++-------- pkg/query-service/interfaces/interface.go | 1 + 6 files changed, 203 insertions(+), 106 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 71388ad1ee..35e5e1732c 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -29,6 +29,7 @@ const ( defaultDependencyGraphTable string = "distributed_dependency_graph_minutes_v2" defaultTopLevelOperationsTable string = "distributed_top_level_operations" defaultSpanAttributeTable string = "distributed_span_attributes" + defaultSpanAttributeKeysTable string = "distributed_span_attributes_keys" defaultLogsDB string = "signoz_logs" defaultLogsTable string = "distributed_logs" defaultLogsLocalTable string = "logs" @@ -65,6 +66,7 @@ type namespaceConfig struct { SpansTable string ErrorTable string SpanAttributeTable string + SpanAttributeKeysTable string DependencyGraphTable string TopLevelOperationsTable string LogsDB string @@ -135,6 +137,7 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s UsageExplorerTable: defaultUsageExplorerTable, SpansTable: defaultSpansTable, SpanAttributeTable: defaultSpanAttributeTable, + SpanAttributeKeysTable: defaultSpanAttributeKeysTable, DependencyGraphTable: defaultDependencyGraphTable, TopLevelOperationsTable: defaultTopLevelOperationsTable, LogsDB: defaultLogsDB, diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 146b5e9377..b75a10bac2 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -96,6 +96,7 @@ type ClickHouseReader struct { usageExplorerTable string SpansTable string spanAttributeTable string + spanAttributesKeysTable string dependencyGraphTable string topLevelOperationsTable string logsDB string @@ -147,6 +148,7 @@ func NewReader(localDB *sqlx.DB, configFile string, featureFlag interfaces.Featu durationTable: options.primary.DurationTable, SpansTable: options.primary.SpansTable, spanAttributeTable: options.primary.SpanAttributeTable, + spanAttributesKeysTable: options.primary.SpanAttributeKeysTable, dependencyGraphTable: options.primary.DependencyGraphTable, topLevelOperationsTable: options.primary.TopLevelOperationsTable, logsDB: options.primary.LogsDB, @@ -4251,7 +4253,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req v3.AggregateOperatorMax: where = "tagKey ILIKE $1 AND dataType='float64'" case - v3.AggregateOpeatorCount, + v3.AggregateOperatorCount, v3.AggregateOperatorNoOp: return &v3.AggregateAttributeResponse{}, nil default: @@ -4380,3 +4382,38 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3. return &attributeValues, nil } + +func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error) { + var query string + var err error + var rows driver.Rows + response := map[string]v3.AttributeKey{} + + query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType, isColumn FROM %s.%s", r.TraceDB, r.spanAttributesKeysTable) + + rows, err = r.db.Query(ctx, query) + + if err != nil { + zap.S().Error(err) + return nil, fmt.Errorf("error while executing query: %s", err.Error()) + } + defer rows.Close() + + var tagKey string + var dataType string + var tagType string + var isColumn bool + for rows.Next() { + if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { + return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) + } + key := v3.AttributeKey{ + Key: tagKey, + DataType: v3.AttributeKeyDataType(dataType), + Type: v3.AttributeKeyType(tagType), + IsColumn: isColumn, + } + response[tagKey] = key + } + return response, nil +} \ No newline at end of file diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 09ba2178ef..e8929552b8 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -2668,6 +2668,20 @@ func (aH *APIHandler) getLogFieldsV3(ctx context.Context, queryRangeParams *v3.Q return data, nil } +func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3) (map[string]v3.AttributeKey, error) { + data := map[string]v3.AttributeKey{} + for _, query := range queryRangeParams.CompositeQuery.BuilderQueries { + if query.DataSource == v3.DataSourceTraces { + spanKeys, err := aH.reader.GetSpanAttributeKeys(ctx) + if err != nil { + return nil, err + } + return spanKeys, nil + } + } + return data, nil +} + func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3, w http.ResponseWriter, r *http.Request) { var result []*v3.Result @@ -2685,7 +2699,15 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que return } - queries, err = aH.queryBuilder.prepareQueries(queryRangeParams, fields) + var spanKeys map[string]v3.AttributeKey + spanKeys, err = aH.getSpanKeysV3(ctx, queryRangeParams) + if err != nil { + apiErrObj := &model.ApiError{Typ: model.ErrorInternal, Err: err} + RespondError(w, apiErrObj, errQuriesByName) + return + } + + queries, err = aH.queryBuilder.prepareQueries(queryRangeParams, fields, spanKeys) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) return diff --git a/pkg/query-service/app/query_builder.go b/pkg/query-service/app/query_builder.go index 59627ed4af..1645496643 100644 --- a/pkg/query-service/app/query_builder.go +++ b/pkg/query-service/app/query_builder.go @@ -35,7 +35,7 @@ var SupportedFunctions = []string{ var evalFuncs = map[string]govaluate.ExpressionFunction{} -type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error) +type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, fields map[string]v3.AttributeKey) (string, error) type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error) @@ -139,7 +139,11 @@ func (qb *queryBuilder) prepareQueries(params *v3.QueryRangeParamsV3, args ...in if query.Expression == queryName { switch query.DataSource { case v3.DataSourceTraces: - queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query) + keys := map[string]v3.AttributeKey{} + if len(args) == 2 { + keys = args[1].(map[string]v3.AttributeKey) + } + queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, keys) if err != nil { return nil, err } diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index a2579f8d4b..58d50626bc 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -1,7 +1,8 @@ -package traces +package v3 import ( "fmt" + "math" "strings" "go.signoz.io/signoz/pkg/query-service/constants" @@ -10,20 +11,15 @@ import ( ) var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ - v3.AggregateOperatorP05: 0.5, - v3.AggregateOperatorP10: 0.10, - v3.AggregateOperatorP20: 0.20, - v3.AggregateOperatorP25: 0.25, - v3.AggregateOperatorP50: 0.50, - v3.AggregateOperatorP75: 0.75, - v3.AggregateOperatorP90: 0.90, - v3.AggregateOperatorP95: 0.95, - v3.AggregateOperatorP99: 0.99, - v3.AggregateOperatorHistQuant50: 0.50, - v3.AggregateOperatorHistQuant75: 0.75, - v3.AggregateOperatorHistQuant90: 0.90, - v3.AggregateOperatorHistQuant95: 0.95, - v3.AggregateOperatorHistQuant99: 0.99, + v3.AggregateOperatorP05: 0.5, + v3.AggregateOperatorP10: 0.10, + v3.AggregateOperatorP20: 0.20, + v3.AggregateOperatorP25: 0.25, + v3.AggregateOperatorP50: 0.50, + v3.AggregateOperatorP75: 0.75, + v3.AggregateOperatorP90: 0.90, + v3.AggregateOperatorP95: 0.95, + v3.AggregateOperatorP99: 0.99, } var aggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{ @@ -55,14 +51,20 @@ var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ v3.FilterOperatorNotExists: "IS NULL", } -func getFilter(key v3.AttributeKey) string { +func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) { + key, err := enrichKeyWithMetadata(key, keys) + if err != nil { + return "", err + } if key.IsColumn { - return key.Key + return key.Key, nil } filterType := key.Type filterDataType := "string" if key.DataType == v3.AttributeKeyDataTypeFloat64 || key.DataType == v3.AttributeKeyDataTypeInt64 { - filterDataType = "float64" + filterDataType = "number" + } else if key.DataType == v3.AttributeKeyDataTypeBool { + filterDataType = "bool" } if filterType == v3.AttributeKeyTypeTag { filterType = "TagMap" @@ -70,31 +72,65 @@ func getFilter(key v3.AttributeKey) string { filterType = "resourceTagsMap" filterDataType = "" } - return fmt.Sprintf("%s%s['%s']", filterType, filterDataType, key.Key) + return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key), nil +} + +func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) (v3.AttributeKey, error) { + if key.Type == "" || key.DataType == "" { + // check if the key is present in the keys map + if existingKey, ok := keys[key.Key]; ok { + key.IsColumn = existingKey.IsColumn + key.Type = existingKey.Type + key.DataType = existingKey.DataType + } else { + return key, fmt.Errorf("key not found to enrich metadata") + } + } + return key, nil } // getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator -func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { +func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) { var selectLabels string if aggregatorOperator == v3.AggregateOperatorNoOp { selectLabels = "" } else { for _, tag := range groupBy { - filterName := getFilter(tag) + filterName, err := getColumnName(tag, keys) + if err != nil { + return "", err + } selectLabels += fmt.Sprintf(", %s as %s", filterName, tag.Key) } } - return selectLabels + return selectLabels, nil +} + +// getZerosForEpochNano returns the number of zeros to be appended to the epoch time for converting it to nanoseconds +func getZerosForEpochNano(epoch int64) int64 { + count := 0 + if epoch == 0 { + count = 1 + } else { + for epoch != 0 { + epoch /= 10 + count++ + } + } + return int64(math.Pow(10, float64(19-count))) } -func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { +func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) (string, error) { var conditions []string if fs != nil && len(fs.Items) != 0 { for _, item := range fs.Items { toFormat := item.Value // generate the key - columnName := getFilter(item.Key) + columnName, err := getColumnName(item.Key, keys) + if err != nil { + return "", err + } fmtVal := utils.ClickHouseFormattedValue(toFormat) if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { @@ -112,16 +148,24 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { return queryString, nil } -func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string) (string, error) { +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey) (string, error) { - filterSubQuery, err := buildTracesFilterQuery(mq.Filters) + filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys) if err != nil { return "", err } + // timerange will be sent in epoch millisecond + spanIndexTableTimeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d')", start*getZerosForEpochNano(start), end*getZerosForEpochNano(end)) - spanIndexTableTimeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d')", start, end) + selectLabels, err := getSelectLabels(mq.AggregateOperator, mq.GroupBy, keys) + if err != nil { + return "", err + } - selectLabels := getSelectLabels(mq.AggregateOperator, mq.GroupBy) + having := having(mq.Having) + if having != "" { + having = " having " + having + } // Select the aggregate value for interval queryTmpl := @@ -129,30 +173,24 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str ", %s as value " + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where " + spanIndexTableTimeFilter + "%s " + - "group by %s " + + "group by %s%s " + "order by %sts" - // tagsWithoutLe is used to group by all tags except le - // This is done because we want to group by le only when we are calculating quantile - // Otherwise, we want to group by all tags except le - // tagsWithoutLe := []string{} - // for _, tag := range mq.GroupBy { - // if tag.Key != "le" { - // tagsWithoutLe = append(tagsWithoutLe, tag.Key) - // } - // } - - // groupByWithoutLe := groupBy(tagsWithoutLe...) - // groupTagsWithoutLe := groupSelect(tagsWithoutLe...) - // orderWithoutLe := orderBy(mq.OrderBy, tagsWithoutLe) - - groupBy := groupByAttributeKeyTags(mq.GroupBy...) - // groupTags := groupSelectAttributeKeyTags(mq.GroupBy...) + groupBy, err := groupByAttributeKeyTags(keys, mq.GroupBy...) + if err != nil { + return "", err + } orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) + if err != nil { + return "", err + } aggregationKey := "" if mq.AggregateAttribute.Key != "" { - aggregationKey = getFilter(mq.AggregateAttribute) + aggregationKey, err = getColumnName(mq.AggregateAttribute, keys) + if err != nil { + return "", err + } } switch mq.AggregateOperator { @@ -161,15 +199,8 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str v3.AggregateOperatorRateAvg, v3.AggregateOperatorRateMin, v3.AggregateOperatorRate: - rateQueryTmpl := - "SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts" + - ", %s as value " + - "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + - " where " + spanIndexTableTimeFilter + "%s " + - "group by %s " + - "order by %sts" op := fmt.Sprintf("%s(%s)/%d", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, step) - query := fmt.Sprintf(rateQueryTmpl, step, op, filterSubQuery, groupBy, orderBy) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorP05, @@ -182,31 +213,26 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str v3.AggregateOperatorP95, v3.AggregateOperatorP99: op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) - query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) - query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCount: op := fmt.Sprintf("toFloat64(count(%s))", aggregationKey) - query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCountDistinct: op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey) - query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, orderBy) + query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) + fmt.Println(query) return query, nil case v3.AggregateOperatorNoOp: - return "", fmt.Errorf("not implemented, part of traces page") - // noOpQueryTmpl := - // "SELECT fingerprint, labels as fullLabels," + - // " toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL %d SECOND) as ts," + - // " any(value) as value" + - // " FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + constants.SIGNOZ_SAMPLES_TABLENAME + - // " GLOBAL INNER JOIN" + - // " (%s)+ ORDER BY ts" - // query := fmt.Sprintf(noOpQueryTmpl, step, filterSubQuery) + // queryTmpl := constants.TracesSQLSelect + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s" + // query := fmt.Sprintf(queryTmpl, spanIndexTableTimeFilter, filterSubQuery) // return query, nil + return "", fmt.Errorf("not implemented, part of traces page") default: return "", fmt.Errorf("unsupported aggregate operator") } @@ -219,29 +245,16 @@ func groupBy(tags ...string) string { return strings.Join(tags, ",") } -// groupSelect returns a string of comma separated tags for select clause -func groupSelect(tags ...string) string { - groupTags := strings.Join(tags, ",") - if len(tags) != 0 { - groupTags += ", " - } - return groupTags -} - -func groupByAttributeKeyTags(tags ...v3.AttributeKey) string { - groupTags := []string{} - for _, tag := range tags { - groupTags = append(groupTags, getFilter(tag)) - } - return groupBy(groupTags...) -} - -func groupSelectAttributeKeyTags(tags ...v3.AttributeKey) string { +func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.AttributeKey) (string, error) { groupTags := []string{} for _, tag := range tags { - groupTags = append(groupTags, getFilter(tag)) + groupTag, err := getColumnName(tag, keys) + if err != nil { + return "", err + } + groupTags = append(groupTags, groupTag) } - return groupSelect(groupTags...) + return groupBy(groupTags...), nil } // orderBy returns a string of comma separated tags for order by clause @@ -261,13 +274,20 @@ func orderBy(items []v3.OrderBy, tags []string) string { orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag)) } } + + // users might want to order by value of aggreagation + for _, item := range items { + if item.ColumnName == constants.SigNozOrderByValue { + orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order)) + } + } return strings.Join(orderBy, ",") } func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string { var groupTags []string for _, tag := range tags { - groupTags = append(groupTags, getFilter(tag)) + groupTags = append(groupTags, tag.Key) } str := orderBy(items, groupTags) if len(str) > 0 { @@ -277,26 +297,17 @@ func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string } func having(items []v3.Having) string { + // aggregate something and filter on that aggregate var having []string for _, item := range items { - having = append(having, fmt.Sprintf("%s %s %v", item.ColumnName, item.Operator, utils.ClickHouseFormattedValue(item.Value))) + having = append(having, fmt.Sprintf("value %s %s", item.Operator, utils.ClickHouseFormattedValue(item.Value))) } return strings.Join(having, " AND ") } func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v3.AggregateOperator) (string, error) { - // var selectLabels string + var groupBy string - // NOOP can possibly return multiple time series and reduce should be applied - // for each uniques series. When the final result contains more than one series we throw - // an error post DB fetching. Otherwise just return the single data. This is not known until queried so the - // the query is prepared accordingly. - // if aggregateOperator == v3.AggregateOperatorNoOp { - // selectLabels = ", any(fullLabels) as fullLabels" - // groupBy = "GROUP BY fingerprint" - // } - // the timestamp picked is not relevant here since the final value used is show the single - // chart with just the query value. For the query switch reduceTo { case v3.ReduceToOperatorLast: query = fmt.Sprintf("SELECT anyLast(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy) @@ -314,13 +325,32 @@ func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator return query, nil } -func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery) (string, error) { - query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME) +func addLimitToQuery(query string, limit uint64, panelType v3.PanelType) string { + if limit == 0 { + limit = 100 + } + if panelType == v3.PanelTypeList { + return fmt.Sprintf("%s LIMIT %d", query, limit) + } + return query +} + +func addOffsetToQuery(query string, offset uint64) string { + return fmt.Sprintf("%s OFFSET %d", query, offset) +} + +func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) { + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys) if err != nil { return "", err } if panelType == v3.PanelTypeValue { query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) } + query = addLimitToQuery(query, mq.Limit, panelType) + + if mq.Offset != 0 { + query = addOffsetToQuery(query, mq.Offset) + } return query, err } diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 1393e28406..cdc1f3387d 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -38,6 +38,7 @@ type Reader interface { GetTraceAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) GetTraceAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) + GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error) GetTagFilters(ctx context.Context, query *model.TagFilterParams) (*model.TagFilters, *model.ApiError) GetTagValues(ctx context.Context, query *model.TagFilterParams) (*model.TagValues, *model.ApiError) GetFilteredSpans(ctx context.Context, query *model.GetFilteredSpansParams) (*model.GetFilterSpansResponse, *model.ApiError) From bd6372d07d3b0f4bd672fe8b125ccff207da8439 Mon Sep 17 00:00:00 2001 From: makeavish Date: Fri, 14 Apr 2023 18:58:01 +0530 Subject: [PATCH 3/8] test: add test cases --- .../app/traces/v3/query_builder.go | 6 +- .../app/traces/v3/query_builder_test.go | 336 +++++++++++++----- 2 files changed, 247 insertions(+), 95 deletions(-) diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index 58d50626bc..3654ddec79 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -131,8 +131,10 @@ func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) ( if err != nil { return "", err } - fmtVal := utils.ClickHouseFormattedValue(toFormat) - + var fmtVal string + if toFormat != "" { + fmtVal = utils.ClickHouseFormattedValue(toFormat) + } if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) } else { diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 9e1b81558d..68408caf73 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -1,103 +1,100 @@ -package traces +package v3 import ( - "fmt" "testing" . "github.com/smartystreets/goconvey/convey" - "github.com/stretchr/testify/require" + "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) -func TestBuildQuery(t *testing.T) { - t.Run("TestSimpleQueryWithName", func(t *testing.T) { - q := &v3.QueryRangeParamsV3{ - Start: 1680066360726210000, - End: 1780066458000000000, - Step: 60, - CompositeQuery: &v3.CompositeQuery{ - BuilderQueries: map[string]*v3.BuilderQuery{ - "A": { - QueryName: "A", - AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true}, - AggregateOperator: v3.AggregateOperatorP20, - Expression: "A", - }, - }, - QueryType: v3.QueryTypeBuilder, - PanelType: v3.PanelTypeGraph, - }, - } - query, err := PrepareTracesQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"]) - require.NoError(t, err) - fmt.Println(query) - // require.Contains(t, query, "WHERE name") - }) -} - -func TestBuildQueryWithFilters(t *testing.T) { - t.Run("TestBuildQueryWithFilters", func(t *testing.T) { - q := &v3.QueryRangeParamsV3{ - Start: 1650991982000, - End: 1651078382000, - Step: 60, - CompositeQuery: &v3.CompositeQuery{ - BuilderQueries: map[string]*v3.BuilderQuery{ - "A": { - QueryName: "A", - AggregateAttribute: v3.AttributeKey{Key: "name"}, - Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "a"}, Value: "b", Operator: "neq"}, - {Key: v3.AttributeKey{Key: "code"}, Value: "ERROR_*", Operator: "nmatch"}, - }}, - AggregateOperator: v3.AggregateOperatorRateMax, - Expression: "A", - }, - }, - }, - } - query, err := PrepareTracesQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"]) - require.NoError(t, err) - - require.Contains(t, query, "WHERE metric_name = 'name' AND JSONExtractString(labels, 'a') != 'b'") - // require.Contains(t, query, rateWithoutNegative) - require.Contains(t, query, "not match(JSONExtractString(labels, 'code'), 'ERROR_*')") - }) +var timeSeriesFilterQueryData = []struct { + Name string + FilterSet *v3.FilterSet + ExpectedFilter string +}{ + { + Name: "Test attribute and resource attribute", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "john", Operator: "="}, + {Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="}, + }}, + ExpectedFilter: " AND stringTagMap['user_name'] = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", + }, + { + Name: "Test materialized column", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "john", Operator: "="}, + {Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="}, + }}, + ExpectedFilter: " AND user_name = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", + }, + { + Name: "Test like", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.%", Operator: "like"}, + }}, + ExpectedFilter: " AND stringTagMap['host'] ILIKE '102.%'", + }, + { + Name: "Test IN", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{1, 2, 3, 4}, Operator: "in"}, + }}, + ExpectedFilter: " AND numberTagMap['bytes'] IN [1,2,3,4]", + }, + { + Name: "Test DataType int64", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: ">"}, + }}, + ExpectedFilter: " AND numberTagMap['bytes'] > 10", + }, + { + Name: "Test NOT IN", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{"john", "bunny"}, Operator: "nin"}, + }}, + ExpectedFilter: " AND stringTagMap['name'] NOT IN ['john','bunny']", + }, + { + Name: "Test exists", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "", Operator: "exists"}, + }}, + ExpectedFilter: " AND stringTagMap['bytes'] IS NOT NULL ", + }, + { + Name: "Test not exists", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "", Operator: "nexists"}, + }}, + ExpectedFilter: " AND stringTagMap['bytes'] IS NULL ", + }, + { + Name: "Test contains", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: "contains"}, + }}, + ExpectedFilter: " AND stringTagMap['host'] ILIKE '%102.%'", + }, + { + Name: "Test not contains", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: "ncontains"}, + }}, + ExpectedFilter: " AND stringTagMap['host'] NOT ILIKE '%102.%'", + }, } -func TestBuildQueryWithMultipleQueries(t *testing.T) { - t.Run("TestBuildQueryWithFilters", func(t *testing.T) { - q := &v3.QueryRangeParamsV3{ - Start: 1650991982000, - End: 1651078382000, - Step: 60, - CompositeQuery: &v3.CompositeQuery{ - BuilderQueries: map[string]*v3.BuilderQuery{ - "A": { - QueryName: "A", - AggregateAttribute: v3.AttributeKey{Key: "name"}, - Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: "in"}, - }}, - AggregateOperator: v3.AggregateOperatorRateAvg, - Expression: "A", - }, - "B": { - QueryName: "B", - AggregateAttribute: v3.AttributeKey{Key: "name2"}, - AggregateOperator: v3.AggregateOperatorRateMax, - Expression: "B", - }, - }, - }, - } - - query, err := PrepareTracesQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"]) - require.NoError(t, err) - - require.Contains(t, query, "WHERE metric_name = 'name' AND JSONExtractString(labels, 'in') IN ['a','b','c']") - // require.Contains(t, query, rateWithoutNegative) - }) +func TestBuildLogsTimeSeriesFilterQuery(t *testing.T) { + for _, tt := range timeSeriesFilterQueryData { + Convey("TestBuildLogsTimeSeriesFilterQuery", t, func() { + query, err := buildTracesFilterQuery(tt.FilterSet, map[string]v3.AttributeKey{}) + So(err, ShouldBeNil) + So(query, ShouldEqual, tt.ExpectedFilter) + }) + } } var testGetFilter = []struct { @@ -108,7 +105,7 @@ var testGetFilter = []struct { { Name: "resource", AttributeKey: v3.AttributeKey{Key: "collector_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: false}, - ExpectedFilter: "resourcesTagMap['collector_id']", + ExpectedFilter: "resourceTagsMap['collector_id']", }, { Name: "stringAttribute", @@ -116,10 +113,20 @@ var testGetFilter = []struct { ExpectedFilter: "stringTagMap['customer_id']", }, { - Name: "numberAttribute", + Name: "boolAttribute", + AttributeKey: v3.AttributeKey{Key: "has_error", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedFilter: "boolTagMap['has_error']", + }, + { + Name: "float64Attribute", AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, ExpectedFilter: "numberTagMap['count']", }, + { + Name: "int64Attribute", + AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedFilter: "numberTagMap['count']", + }, { Name: "column", AttributeKey: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -130,8 +137,151 @@ var testGetFilter = []struct { func TestGetFilter(t *testing.T) { for _, tt := range testGetFilter { Convey("testGetFilter", t, func() { - Filter := getFilter(tt.AttributeKey) + Filter, err := getColumnName(tt.AttributeKey, map[string]v3.AttributeKey{}) + So(err, ShouldBeNil) So(Filter, ShouldEqual, tt.ExpectedFilter) }) } } + +var testGetSelectLabelsData = []struct { + Name string + AggregateOperator v3.AggregateOperator + GroupByTags []v3.AttributeKey + SelectLabels string +}{ + { + Name: "select fields for groupBy attribute", + AggregateOperator: v3.AggregateOperatorCount, + GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + SelectLabels: ", stringTagMap['user_name'] as user_name", + }, + { + Name: "select fields for groupBy resource", + AggregateOperator: v3.AggregateOperatorCount, + GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, + SelectLabels: ", resourceTagsMap['user_name'] as user_name", + }, + { + Name: "select fields for groupBy attribute and resource", + AggregateOperator: v3.AggregateOperatorCount, + GroupByTags: []v3.AttributeKey{ + {Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + {Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + SelectLabels: ", resourceTagsMap['user_name'] as user_name, stringTagMap['host'] as host", + }, + { + Name: "select fields for groupBy materialized columns", + AggregateOperator: v3.AggregateOperatorCount, + GroupByTags: []v3.AttributeKey{{Key: "host", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + SelectLabels: ", host as host", + }, +} + +func TestGetSelectLabels(t *testing.T) { + for _, tt := range testGetSelectLabelsData { + Convey("testGetSelectLabelsData", t, func() { + selectLabels, err := getSelectLabels(tt.AggregateOperator, tt.GroupByTags, map[string]v3.AttributeKey{}) + So(err, ShouldBeNil) + So(selectLabels, ShouldEqual, tt.SelectLabels) + }) + } +} + +var testGetZerosForEpochNanoData = []struct { + Name string + Epoch int64 + Multiplier int64 + Result int64 +}{ + { + Name: "Test 1", + Epoch: 1680712080000, + Multiplier: 1000000, + Result: 1680712080000000000, + }, + { + Name: "Test 2", + Epoch: 1680712080000000000, + Multiplier: 1, + Result: 1680712080000000000, + }, +} + +func TestGetZerosForEpochNano(t *testing.T) { + for _, tt := range testGetZerosForEpochNanoData { + Convey("testGetZerosForEpochNanoData", t, func() { + multiplier := getZerosForEpochNano(tt.Epoch) + So(multiplier, ShouldEqual, tt.Multiplier) + So(tt.Epoch*multiplier, ShouldEqual, tt.Result) + }) + } +} + +var testOrderBy = []struct { + Name string + Items []v3.OrderBy + Tags []string + Result string +}{ + { + Name: "Test 1", + Items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: constants.SigNozOrderByValue, + Order: "desc", + }, + }, + Tags: []string{"name"}, + Result: "name asc,value desc", + }, + { + Name: "Test 2", + Items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: "bytes", + Order: "asc", + }, + }, + Tags: []string{"name", "bytes"}, + Result: "name asc,bytes asc", + }, + { + Name: "Test 3", + Items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: constants.SigNozOrderByValue, + Order: "asc", + }, + { + ColumnName: "bytes", + Order: "asc", + }, + }, + Tags: []string{"name", "bytes"}, + Result: "name asc,bytes asc,value asc", + }, +} + +func TestOrderBy(t *testing.T) { + for _, tt := range testOrderBy { + Convey("testOrderBy", t, func() { + res := orderBy(tt.Items, tt.Tags) + So(res, ShouldEqual, tt.Result) + }) + } +} + From 2b467435d2e474053bbff0f2bb4cc0631527a9fe Mon Sep 17 00:00:00 2001 From: makeavish Date: Mon, 17 Apr 2023 12:05:57 +0530 Subject: [PATCH 4/8] test: add more tests and handle few edge cases --- .../app/traces/v3/query_builder.go | 41 +- .../app/traces/v3/query_builder_test.go | 449 +++++++++++++++++- 2 files changed, 463 insertions(+), 27 deletions(-) diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index 3654ddec79..c32c685fdd 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -11,7 +11,7 @@ import ( ) var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ - v3.AggregateOperatorP05: 0.5, + v3.AggregateOperatorP05: 0.05, v3.AggregateOperatorP10: 0.10, v3.AggregateOperatorP20: 0.20, v3.AggregateOperatorP25: 0.25, @@ -47,8 +47,8 @@ var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ v3.FilterOperatorNotLike: "NOT ILIKE", v3.FilterOperatorContains: "ILIKE", v3.FilterOperatorNotContains: "NOT ILIKE", - v3.FilterOperatorExists: "IS NOT NULL", - v3.FilterOperatorNotExists: "IS NULL", + v3.FilterOperatorExists: "has(%s%s, '%s')", + v3.FilterOperatorNotExists: "NOT has(%s%s, '%s')", } func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) { @@ -59,6 +59,11 @@ func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) (string if key.IsColumn { return key.Key, nil } + filterType, filterDataType := getClickhouseTracesColumnDataTypeAndType(key) + return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key), nil +} + +func getClickhouseTracesColumnDataTypeAndType(key v3.AttributeKey) (v3.AttributeKeyType, string) { filterType := key.Type filterDataType := "string" if key.DataType == v3.AttributeKeyDataTypeFloat64 || key.DataType == v3.AttributeKeyDataTypeInt64 { @@ -72,7 +77,7 @@ func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) (string filterType = "resourceTagsMap" filterDataType = "" } - return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key), nil + return filterType, filterDataType } func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) (v3.AttributeKey, error) { @@ -100,7 +105,7 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri if err != nil { return "", err } - selectLabels += fmt.Sprintf(", %s as %s", filterName, tag.Key) + selectLabels += fmt.Sprintf(", %s as `%s`", filterName, tag.Key) } } return selectLabels, nil @@ -136,7 +141,21 @@ func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) ( fmtVal = utils.ClickHouseFormattedValue(toFormat) } if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { - conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) + switch item.Operator { + case v3.FilterOperatorContains, v3.FilterOperatorNotContains: + conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, item.Value)) + + case v3.FilterOperatorExists, v3.FilterOperatorNotExists: + key, err := enrichKeyWithMetadata(item.Key, keys) + if err != nil { + return "", err + } + columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key) + conditions = append(conditions, fmt.Sprintf(operator, columnDataType, columnType, item.Key.Key)) + + default: + conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) + } } else { return "", fmt.Errorf("unsupported operation") } @@ -222,7 +241,15 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCount: - op := fmt.Sprintf("toFloat64(count(%s))", aggregationKey) + if mq.AggregateAttribute.Key != "" { + key, err := enrichKeyWithMetadata(mq.AggregateAttribute, keys) + if err != nil { + return "", err + } + columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key) + filterSubQuery = fmt.Sprintf("%s AND has(%s%s, '%s')", filterSubQuery, columnDataType, columnType, mq.AggregateAttribute.Key) + } + op := "toFloat64(count())" query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCountDistinct: diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 68408caf73..25296045cb 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -16,18 +16,18 @@ var timeSeriesFilterQueryData = []struct { { Name: "Test attribute and resource attribute", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "john", Operator: "="}, + {Key: v3.AttributeKey{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "john", Operator: "="}, {Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="}, }}, - ExpectedFilter: " AND stringTagMap['user_name'] = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", + ExpectedFilter: " AND stringTagMap['user.name'] = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", }, { - Name: "Test materialized column", + Name: "Test fixed column", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "john", Operator: "="}, + {Key: v3.AttributeKey{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "john", Operator: "="}, {Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="}, }}, - ExpectedFilter: " AND user_name = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", + ExpectedFilter: " AND user.name = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", }, { Name: "Test like", @@ -62,14 +62,14 @@ var timeSeriesFilterQueryData = []struct { FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "", Operator: "exists"}, }}, - ExpectedFilter: " AND stringTagMap['bytes'] IS NOT NULL ", + ExpectedFilter: " AND has(stringTagMap, 'bytes')", }, { Name: "Test not exists", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "", Operator: "nexists"}, }}, - ExpectedFilter: " AND stringTagMap['bytes'] IS NULL ", + ExpectedFilter: " AND NOT has(stringTagMap, 'bytes')", }, { Name: "Test contains", @@ -87,9 +87,9 @@ var timeSeriesFilterQueryData = []struct { }, } -func TestBuildLogsTimeSeriesFilterQuery(t *testing.T) { +func TestBuildTracesTimeSeriesFilterQuery(t *testing.T) { for _, tt := range timeSeriesFilterQueryData { - Convey("TestBuildLogsTimeSeriesFilterQuery", t, func() { + Convey("TestBuildTracesTimeSeriesFilterQuery", t, func() { query, err := buildTracesFilterQuery(tt.FilterSet, map[string]v3.AttributeKey{}) So(err, ShouldBeNil) So(query, ShouldEqual, tt.ExpectedFilter) @@ -151,31 +151,31 @@ var testGetSelectLabelsData = []struct { SelectLabels string }{ { - Name: "select fields for groupBy attribute", + Name: "select keys for groupBy attribute", AggregateOperator: v3.AggregateOperatorCount, - GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - SelectLabels: ", stringTagMap['user_name'] as user_name", + GroupByTags: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + SelectLabels: ", stringTagMap['user.name'] as `user.name`", }, { - Name: "select fields for groupBy resource", + Name: "select keys for groupBy resource", AggregateOperator: v3.AggregateOperatorCount, - GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, - SelectLabels: ", resourceTagsMap['user_name'] as user_name", + GroupByTags: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, + SelectLabels: ", resourceTagsMap['user.name'] as `user.name`", }, { - Name: "select fields for groupBy attribute and resource", + Name: "select keys for groupBy attribute and resource", AggregateOperator: v3.AggregateOperatorCount, GroupByTags: []v3.AttributeKey{ - {Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + {Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, {Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, }, - SelectLabels: ", resourceTagsMap['user_name'] as user_name, stringTagMap['host'] as host", + SelectLabels: ", resourceTagsMap['user.name'] as `user.name`, stringTagMap['host'] as `host`", }, { - Name: "select fields for groupBy materialized columns", + Name: "select keys for groupBy fixed columns", AggregateOperator: v3.AggregateOperatorCount, GroupByTags: []v3.AttributeKey{{Key: "host", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - SelectLabels: ", host as host", + SelectLabels: ", host as `host`", }, } @@ -285,3 +285,412 @@ func TestOrderBy(t *testing.T) { } } +var testBuildTracesQueryData = []struct { + Name string + Start int64 + End int64 + Step int64 + BuilderQuery *v3.BuilderQuery + GroupByTags []v3.AttributeKey + TableName string + AggregateOperator v3.AggregateOperator + ExpectedQuery string +}{ + { + Name: "Test aggregate count on column", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + }, + { + Name: "Test aggregate count on a attribute", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND has(stringTagMap, 'user_name') group by ts order by ts", + }, + { + Name: "Test aggregate count on a with filter", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCount, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: ">"}, + }}, + Expression: "A", + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND numberTagMap['bytes'] > 100 AND has(stringTagMap, 'user_name') group by ts order by ts", + }, + { + Name: "Test aggregate count distinct and order by value", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by value ASC,ts", + }, + { + Name: "Test aggregate count distinct on non selected field", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + }, + { + Name: "Test aggregate count distinct with filter and groupBy", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + {Key: v3.AttributeKey{Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "abc", Operator: "!="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "http.method", Order: "ASC"}, {ColumnName: "ts", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['http.method'] as `http.method`, " + + "toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + + "group by stringTagMap['http.method'],ts " + + "order by http.method ASC,ts", + }, + { + Name: "Test aggregate count with multiple filter,groupBy and orderBy", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + {Key: v3.AttributeKey{Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "abc", Operator: "!="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, {Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "x", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['method'] as `method`, " + + "resourceTagsMap['x'] as `x`, " + + "toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + + "group by stringTagMap['method'],resourceTagsMap['x'],ts " + + "order by method ASC,x ASC,ts", + }, + { + Name: "Test aggregate avg", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorAvg, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "x", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['method'] as `method`, " + + "avg(numberTagMap['bytes']) as value " + + "from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate sum", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorSum, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['method'] as `method`, " + + "sum(bytes) as value " + + "from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate min", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorMin, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['method'] as `method`, " + + "min(bytes) as value " + + "from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate max", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorMax, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['method'] as `method`, " + + "max(bytes) as value " + + "from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate PXX", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorP05, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + + " stringTagMap['method'] as `method`, " + + "quantile(0.05)(bytes) as value " + + "from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate RateSum", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorRateSum, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + + ", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " group by stringTagMap['method'],ts order by method ASC,ts", + }, + { + Name: "Test aggregate rate", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64}, + AggregateOperator: v3.AggregateOperatorRate, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + + ", count(numberTagMap['bytes'])/60 as value " + + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate RateSum without fixed column", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64}, + AggregateOperator: v3.AggregateOperatorRateSum, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + + "stringTagMap['method'] as `method`, " + + "sum(numberTagMap['bytes'])/60 as value " + + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "group by stringTagMap['method'],ts " + + "order by method ASC,ts", + }, + { + Name: "Test aggregate with having clause", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Having: []v3.Having{ + { + ColumnName: "name", + Operator: ">", + Value: 10, + }, + }, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts having value > 10 order by ts", + }, + { + Name: "Test aggregate with having clause and filters", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + Having: []v3.Having{ + { + ColumnName: "name", + Operator: ">", + Value: 10, + }, + }, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts", + }, + // { + // Name: "Test Noop", + // Start: 1680066360726210000, + // End: 1680066458000000000, + // Step: 60, + // BuilderQuery: &v3.BuilderQuery{ + // SelectColumns: []v3.AttributeKey{}, + // QueryName: "A", + // AggregateOperator: v3.AggregateOperatorNoOp, + // Expression: "A", + // Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + // // GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + // // OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + // }, + // ExpectedQuery: "", + // }, +} + +func TestBuildTracesQuery(t *testing.T) { + for _, tt := range testBuildTracesQueryData { + Convey("TestBuildTracesQuery", t, func() { + query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{}) + So(err, ShouldBeNil) + So(query, ShouldEqual, tt.ExpectedQuery) + + }) + } +} From a31fcf783a93b0e53d23b340994168a743883b6c Mon Sep 17 00:00:00 2001 From: makeavish Date: Mon, 17 Apr 2023 12:12:32 +0530 Subject: [PATCH 5/8] test: add test for count distinct aggregate --- .../app/traces/v3/query_builder_test.go | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 25296045cb..6adc1d7749 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -640,7 +640,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts having value > 10 order by ts", }, { - Name: "Test aggregate with having clause and filters", + Name: "Test count aggregate with having clause and filters", Start: 1680066360726210000, End: 1680066458000000000, Step: 60, @@ -662,6 +662,33 @@ var testBuildTracesQueryData = []struct { }, }, TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from " + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts", + }, + { + Name: "Test count distinct aggregate with having clause and filters", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + Having: []v3.Having{ + { + ColumnName: "name", + Operator: ">", + Value: 10, + }, + }, + }, + TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts", From b28fdd949c7b5931ad8c2f66745ab29580a77bd2 Mon Sep 17 00:00:00 2001 From: makeavish Date: Tue, 18 Apr 2023 16:08:17 +0530 Subject: [PATCH 6/8] test: update test --- pkg/query-service/app/metrics/v3/query_builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query-service/app/metrics/v3/query_builder_test.go b/pkg/query-service/app/metrics/v3/query_builder_test.go index c661c53fad..bcb9cfc69e 100644 --- a/pkg/query-service/app/metrics/v3/query_builder_test.go +++ b/pkg/query-service/app/metrics/v3/query_builder_test.go @@ -18,7 +18,7 @@ func TestBuildQuery(t *testing.T) { "A": { QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name"}, - AggregateOperator: v3.AggregateOperatorNoOp, + AggregateOperator: v3.AggregateOperatorRateMax, Expression: "A", }, }, From aa772ceb85fd983a2a966f14b7bd0ab67c360e6c Mon Sep 17 00:00:00 2001 From: makeavish Date: Wed, 19 Apr 2023 10:11:33 +0530 Subject: [PATCH 7/8] fix: few edge cases found in review --- .../app/traces/v3/query_builder.go | 23 ++------ .../app/traces/v3/query_builder_test.go | 57 ++++++++++++++----- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index c32c685fdd..305a7e3e9b 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -137,9 +137,7 @@ func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) ( return "", err } var fmtVal string - if toFormat != "" { - fmtVal = utils.ClickHouseFormattedValue(toFormat) - } + fmtVal = utils.ClickHouseFormattedValue(toFormat) if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { switch item.Operator { case v3.FilterOperatorContains, v3.FilterOperatorNotContains: @@ -197,14 +195,8 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str "group by %s%s " + "order by %sts" - groupBy, err := groupByAttributeKeyTags(keys, mq.GroupBy...) - if err != nil { - return "", err - } + groupBy := groupByAttributeKeyTags(keys, mq.GroupBy...) orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) - if err != nil { - return "", err - } aggregationKey := "" if mq.AggregateAttribute.Key != "" { @@ -255,7 +247,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str case v3.AggregateOperatorCountDistinct: op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey) query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) - fmt.Println(query) return query, nil case v3.AggregateOperatorNoOp: // queryTmpl := constants.TracesSQLSelect + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s" @@ -274,16 +265,12 @@ func groupBy(tags ...string) string { return strings.Join(tags, ",") } -func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.AttributeKey) (string, error) { +func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.AttributeKey) string { groupTags := []string{} for _, tag := range tags { - groupTag, err := getColumnName(tag, keys) - if err != nil { - return "", err - } - groupTags = append(groupTags, groupTag) + groupTags = append(groupTags, tag.Key) } - return groupBy(groupTags...), nil + return groupBy(groupTags...) } // orderBy returns a string of comma separated tags for order by clause diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 6adc1d7749..89c54d3169 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -8,7 +8,7 @@ import ( v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) -var timeSeriesFilterQueryData = []struct { +var buildFilterQueryData = []struct { Name string FilterSet *v3.FilterSet ExpectedFilter string @@ -29,6 +29,14 @@ var timeSeriesFilterQueryData = []struct { }}, ExpectedFilter: " AND user.name = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'", }, + { + Name: "Test fixed column with empty value", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "="}, + {Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="}, + }}, + ExpectedFilter: " AND user.name = '' AND resourceTagsMap['k8s_namespace'] != 'my_service'", + }, { Name: "Test like", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -64,6 +72,13 @@ var timeSeriesFilterQueryData = []struct { }}, ExpectedFilter: " AND has(stringTagMap, 'bytes')", }, + { + Name: "Test exists with fixed column", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "exists"}, + }}, + ExpectedFilter: " AND isNotNull(name)", + }, { Name: "Test not exists", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -87,9 +102,9 @@ var timeSeriesFilterQueryData = []struct { }, } -func TestBuildTracesTimeSeriesFilterQuery(t *testing.T) { - for _, tt := range timeSeriesFilterQueryData { - Convey("TestBuildTracesTimeSeriesFilterQuery", t, func() { +func TestBuildTracesFilterQuery(t *testing.T) { + for _, tt := range buildFilterQueryData { + Convey("TestBuildTracesFilterQuery", t, func() { query, err := buildTracesFilterQuery(tt.FilterSet, map[string]v3.AttributeKey{}) So(err, ShouldBeNil) So(query, ShouldEqual, tt.ExpectedFilter) @@ -323,6 +338,20 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND has(stringTagMap, 'user_name') group by ts order by ts", }, + { + Name: "Test aggregate count on a fixed column", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + AggregateOperator: v3.AggregateOperatorCount, + Expression: "A", + }, + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + }, { Name: "Test aggregate count on a with filter", Start: 1680066360726210000, @@ -393,7 +422,7 @@ var testBuildTracesQueryData = []struct { "toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + - "group by stringTagMap['http.method'],ts " + + "group by http.method,ts " + "order by http.method ASC,ts", }, { @@ -421,7 +450,7 @@ var testBuildTracesQueryData = []struct { "toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + - "group by stringTagMap['method'],resourceTagsMap['x'],ts " + + "group by method,x,ts " + "order by method ASC,x ASC,ts", }, { @@ -448,7 +477,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { @@ -475,7 +504,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { @@ -502,7 +531,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { @@ -529,7 +558,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { @@ -552,7 +581,7 @@ var testBuildTracesQueryData = []struct { "quantile(0.05)(bytes) as value " + "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { @@ -573,7 +602,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + ", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " group by stringTagMap['method'],ts order by method ASC,ts", + " group by method,ts order by method ASC,ts", }, { Name: "Test aggregate rate", @@ -593,7 +622,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + ", count(numberTagMap['bytes'])/60 as value " + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { @@ -615,7 +644,7 @@ var testBuildTracesQueryData = []struct { "stringTagMap['method'] as `method`, " + "sum(numberTagMap['bytes'])/60 as value " + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "group by stringTagMap['method'],ts " + + "group by method,ts " + "order by method ASC,ts", }, { From 0980d5f1cd69a33741b72c0e6a34948974f415b4 Mon Sep 17 00:00:00 2001 From: makeavish Date: Tue, 25 Apr 2023 00:51:02 +0530 Subject: [PATCH 8/8] chore: handle edge cases, review comments --- .../app/traces/v3/query_builder.go | 121 +++++++++----- .../app/traces/v3/query_builder_test.go | 148 +++++++++++++----- pkg/query-service/utils/format.go | 1 + 3 files changed, 192 insertions(+), 78 deletions(-) diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index 305a7e3e9b..5f44b449a3 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -51,16 +51,13 @@ var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ v3.FilterOperatorNotExists: "NOT has(%s%s, '%s')", } -func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) { - key, err := enrichKeyWithMetadata(key, keys) - if err != nil { - return "", err - } +func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) string { + key = enrichKeyWithMetadata(key, keys) if key.IsColumn { - return key.Key, nil + return key.Key } filterType, filterDataType := getClickhouseTracesColumnDataTypeAndType(key) - return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key), nil + return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key) } func getClickhouseTracesColumnDataTypeAndType(key v3.AttributeKey) (v3.AttributeKeyType, string) { @@ -80,18 +77,21 @@ func getClickhouseTracesColumnDataTypeAndType(key v3.AttributeKey) (v3.Attribute return filterType, filterDataType } -func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) (v3.AttributeKey, error) { +func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey { if key.Type == "" || key.DataType == "" { // check if the key is present in the keys map if existingKey, ok := keys[key.Key]; ok { key.IsColumn = existingKey.IsColumn key.Type = existingKey.Type key.DataType = existingKey.DataType - } else { - return key, fmt.Errorf("key not found to enrich metadata") + } else { // if not present then set the default values + key.Type = v3.AttributeKeyTypeTag + key.DataType = v3.AttributeKeyDataTypeString + key.IsColumn = false + return key } } - return key, nil + return key } // getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator @@ -101,10 +101,7 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri selectLabels = "" } else { for _, tag := range groupBy { - filterName, err := getColumnName(tag, keys) - if err != nil { - return "", err - } + filterName := getColumnName(tag, keys) selectLabels += fmt.Sprintf(", %s as `%s`", filterName, tag.Key) } } @@ -130,32 +127,41 @@ func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) ( if fs != nil && len(fs.Items) != 0 { for _, item := range fs.Items { - toFormat := item.Value + val := item.Value // generate the key - columnName, err := getColumnName(item.Key, keys) - if err != nil { - return "", err - } + columnName := getColumnName(item.Key, keys) var fmtVal string - fmtVal = utils.ClickHouseFormattedValue(toFormat) + key := enrichKeyWithMetadata(item.Key, keys) + if item.Operator != v3.FilterOperatorExists && item.Operator != v3.FilterOperatorNotExists { + var err error + val, err = utils.ValidateAndCastValue(val, key.DataType) + if err != nil { + return "", fmt.Errorf("invalid value for key %s: %v", item.Key.Key, err) + } + } + fmtVal = utils.ClickHouseFormattedValue(val) if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { switch item.Operator { case v3.FilterOperatorContains, v3.FilterOperatorNotContains: conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, item.Value)) case v3.FilterOperatorExists, v3.FilterOperatorNotExists: - key, err := enrichKeyWithMetadata(item.Key, keys) - if err != nil { - return "", err + if key.IsColumn { + subQuery, err := existsSubQueryForFixedColumn(key, item.Operator) + if err != nil { + return "", err + } + conditions = append(conditions, subQuery) + } else { + columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key) + conditions = append(conditions, fmt.Sprintf(operator, columnDataType, columnType, key.Key)) } - columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key) - conditions = append(conditions, fmt.Sprintf(operator, columnDataType, columnType, item.Key.Key)) default: conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) } } else { - return "", fmt.Errorf("unsupported operation") + return "", fmt.Errorf("unsupported operator %s", item.Operator) } } } @@ -167,6 +173,41 @@ func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) ( return queryString, nil } +func existsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) { + if key.DataType == v3.AttributeKeyDataTypeString { + if op == v3.FilterOperatorExists { + return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorNotEqual]), nil + } else { + return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorEqual]), nil + } + } else { + return "", fmt.Errorf("unsupported operation, exists and not exists can only be applied on custom attributes or string type columns") + } +} + +func handleEmptyValuesInGroupBy(keys map[string]v3.AttributeKey, groupBy []v3.AttributeKey) (string, error) { + filterItems := []v3.FilterItem{} + if len(groupBy) != 0 { + for _, item := range groupBy { + key := enrichKeyWithMetadata(item, keys) + if !key.IsColumn { + filterItems = append(filterItems, v3.FilterItem{ + Key: item, + Operator: v3.FilterOperatorExists, + }) + } + } + } + if len(filterItems) != 0 { + filterSet := v3.FilterSet{ + Operator: "AND", + Items: filterItems, + } + return buildTracesFilterQuery(&filterSet, keys) + } + return "", nil +} + func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey) (string, error) { filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys) @@ -195,15 +236,18 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str "group by %s%s " + "order by %sts" + emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy) + if err != nil { + return "", err + } + filterSubQuery += emptyValuesInGroupByFilter + groupBy := groupByAttributeKeyTags(keys, mq.GroupBy...) orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) aggregationKey := "" if mq.AggregateAttribute.Key != "" { - aggregationKey, err = getColumnName(mq.AggregateAttribute, keys) - if err != nil { - return "", err - } + aggregationKey = getColumnName(mq.AggregateAttribute, keys) } switch mq.AggregateOperator { @@ -234,12 +278,17 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str return query, nil case v3.AggregateOperatorCount: if mq.AggregateAttribute.Key != "" { - key, err := enrichKeyWithMetadata(mq.AggregateAttribute, keys) - if err != nil { - return "", err + key := enrichKeyWithMetadata(mq.AggregateAttribute, keys) + if key.IsColumn { + subQuery, err := existsSubQueryForFixedColumn(key, v3.FilterOperatorExists) + if err != nil { + filterSubQuery = "" + } + filterSubQuery = fmt.Sprintf(" AND %s", subQuery) + } else { + columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key) + filterSubQuery = fmt.Sprintf("%s AND has(%s%s, '%s')", filterSubQuery, columnDataType, columnType, mq.AggregateAttribute.Key) } - columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key) - filterSubQuery = fmt.Sprintf("%s AND has(%s%s, '%s')", filterSubQuery, columnDataType, columnType, mq.AggregateAttribute.Key) } op := "toFloat64(count())" query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 89c54d3169..83a307caae 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -77,7 +77,7 @@ var buildFilterQueryData = []struct { FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "exists"}, }}, - ExpectedFilter: " AND isNotNull(name)", + ExpectedFilter: " AND name != ''", }, { Name: "Test not exists", @@ -86,6 +86,13 @@ var buildFilterQueryData = []struct { }}, ExpectedFilter: " AND NOT has(stringTagMap, 'bytes')", }, + { + Name: "Test not exists with fixed column", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "nexists"}, + }}, + ExpectedFilter: " AND name = ''", + }, { Name: "Test contains", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -112,49 +119,89 @@ func TestBuildTracesFilterQuery(t *testing.T) { } } -var testGetFilter = []struct { +var handleEmptyValuesInGroupByData = []struct { Name string - AttributeKey v3.AttributeKey + GroupBy []v3.AttributeKey ExpectedFilter string +}{ + { + Name: "String type key", + GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + ExpectedFilter: " AND has(stringTagMap, 'bytes')", + }, + { + Name: "fixed column type key", + GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + ExpectedFilter: "", + }, + { + Name: "String, float64 and fixed column type key", + GroupBy: []v3.AttributeKey{ + {Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + ExpectedFilter: " AND has(stringTagMap, 'bytes') AND has(numberTagMap, 'count')", + }, +} + +func TestBuildTracesHandleEmptyValuesInGroupBy(t *testing.T) { + for _, tt := range handleEmptyValuesInGroupByData { + Convey("TestBuildTracesHandleEmptyValuesInGroupBy", t, func() { + query, err := handleEmptyValuesInGroupBy(map[string]v3.AttributeKey{}, tt.GroupBy) + So(err, ShouldBeNil) + So(query, ShouldEqual, tt.ExpectedFilter) + }) + } +} + +var testColumnName = []struct { + Name string + AttributeKey v3.AttributeKey + ExpectedColumn string }{ { Name: "resource", AttributeKey: v3.AttributeKey{Key: "collector_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: false}, - ExpectedFilter: "resourceTagsMap['collector_id']", + ExpectedColumn: "resourceTagsMap['collector_id']", }, { Name: "stringAttribute", AttributeKey: v3.AttributeKey{Key: "customer_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedFilter: "stringTagMap['customer_id']", + ExpectedColumn: "stringTagMap['customer_id']", }, { Name: "boolAttribute", AttributeKey: v3.AttributeKey{Key: "has_error", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedFilter: "boolTagMap['has_error']", + ExpectedColumn: "boolTagMap['has_error']", }, { Name: "float64Attribute", AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedFilter: "numberTagMap['count']", + ExpectedColumn: "numberTagMap['count']", }, { Name: "int64Attribute", AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedFilter: "numberTagMap['count']", + ExpectedColumn: "numberTagMap['count']", }, { Name: "column", AttributeKey: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, - ExpectedFilter: "name", + ExpectedColumn: "name", + }, + { + Name: "missing key", + AttributeKey: v3.AttributeKey{Key: "xyz"}, + ExpectedColumn: "stringTagMap['xyz']", }, } -func TestGetFilter(t *testing.T) { - for _, tt := range testGetFilter { - Convey("testGetFilter", t, func() { - Filter, err := getColumnName(tt.AttributeKey, map[string]v3.AttributeKey{}) - So(err, ShouldBeNil) - So(Filter, ShouldEqual, tt.ExpectedFilter) +func TestColumnName(t *testing.T) { + for _, tt := range testColumnName { + Convey("testColumnName", t, func() { + Column := getColumnName(tt.AttributeKey, map[string]v3.AttributeKey{}) + So(Column, ShouldEqual, tt.ExpectedColumn) }) } } @@ -321,8 +368,10 @@ var testBuildTracesQueryData = []struct { AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " group by ts order by ts", }, { Name: "Test aggregate count on a attribute", @@ -335,8 +384,10 @@ var testBuildTracesQueryData = []struct { AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND has(stringTagMap, 'user_name') group by ts order by ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " AND has(stringTagMap, 'user_name') group by ts order by ts", }, { Name: "Test aggregate count on a fixed column", @@ -349,11 +400,13 @@ var testBuildTracesQueryData = []struct { AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " AND name != '' group by ts order by ts", }, { - Name: "Test aggregate count on a with filter", + Name: "Test aggregate count with filter", Start: 1680066360726210000, End: 1680066458000000000, Step: 60, @@ -366,8 +419,10 @@ var testBuildTracesQueryData = []struct { }}, Expression: "A", }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND numberTagMap['bytes'] > 100 AND has(stringTagMap, 'user_name') group by ts order by ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by ts", }, { Name: "Test aggregate count distinct and order by value", @@ -381,11 +436,13 @@ var testBuildTracesQueryData = []struct { Expression: "A", OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by value ASC,ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " group by ts order by value ASC,ts", }, { - Name: "Test aggregate count distinct on non selected field", + Name: "Test aggregate count distinct on string key", Start: 1680066360726210000, End: 1680066458000000000, Step: 60, @@ -395,8 +452,10 @@ var testBuildTracesQueryData = []struct { AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name'])))" + + " as value from signoz_traces.distributed_signoz_index_v2 where" + + " (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", }, { Name: "Test aggregate count distinct with filter and groupBy", @@ -422,7 +481,7 @@ var testBuildTracesQueryData = []struct { "toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + - "group by http.method,ts " + + "AND has(stringTagMap, 'http.method') group by http.method,ts " + "order by http.method ASC,ts", }, { @@ -440,7 +499,10 @@ var testBuildTracesQueryData = []struct { {Key: v3.AttributeKey{Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "abc", Operator: "!="}, }, }, - GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, {Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, + GroupBy: []v3.AttributeKey{ + {Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + }, OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "x", Order: "ASC"}}, }, TableName: "signoz_traces.distributed_signoz_index_v2", @@ -450,7 +512,7 @@ var testBuildTracesQueryData = []struct { "toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + - "group by method,x,ts " + + "AND has(stringTagMap, 'method') AND has(resourceTagsMap, 'x') group by method,x,ts " + "order by method ASC,x ASC,ts", }, { @@ -477,7 +539,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -504,7 +566,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -531,7 +593,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -558,7 +620,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -581,7 +643,7 @@ var testBuildTracesQueryData = []struct { "quantile(0.05)(bytes) as value " + "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -602,7 +664,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + ", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " group by method,ts order by method ASC,ts", + " AND has(stringTagMap, 'method') group by method,ts order by method ASC,ts", }, { Name: "Test aggregate rate", @@ -622,7 +684,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + ", count(numberTagMap['bytes'])/60 as value " + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -644,7 +706,7 @@ var testBuildTracesQueryData = []struct { "stringTagMap['method'] as `method`, " + "sum(numberTagMap['bytes'])/60 as value " + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "group by method,ts " + + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", }, { @@ -665,8 +727,10 @@ var testBuildTracesQueryData = []struct { }, }, }, - TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts having value > 10 order by ts", + TableName: "signoz_traces.distributed_signoz_index_v2", + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " group by ts having value > 10 order by ts", }, { Name: "Test count aggregate with having clause and filters", diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go index 3316205966..7571e900d4 100644 --- a/pkg/query-service/utils/format.go +++ b/pkg/query-service/utils/format.go @@ -10,6 +10,7 @@ import ( "go.uber.org/zap" ) +// ValidateAndCastValue validates and casts the value of a key to the corresponding data type of the key func ValidateAndCastValue(v interface{}, dataType v3.AttributeKeyDataType) (interface{}, error) { switch dataType { case v3.AttributeKeyDataTypeString: