Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Tabular result format #239

Merged
merged 8 commits into from
Sep 17, 2024
Merged

Tabular result format #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a9b46f6
feat(query): tabular result format
lukasmalkmus Jan 24, 2023
6debfe2
feat(query): iterator for columns and rows
lukasmalkmus Jan 24, 2023
d3ef537
feat(query): more typing
lukasmalkmus Feb 1, 2023
7b63da4
feat(query): go 1.23 iterator support
lukasmalkmus Aug 10, 2024
efd08b8
tabular: conform to DB type fields
dasfmi Aug 15, 2024
792d60e
omnia fixes
lukasmalkmus Sep 17, 2024
9c7d7ca
deps: update
lukasmalkmus Sep 17, 2024
9e33ffb
fix example
lukasmalkmus Sep 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test_examples.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ jobs:
if: matrix.setup
run: ${{ matrix.setup }}
- name: Run example
run: go run ./examples/${{ matrix.example }}/main.go
run: go run ./examples/${{ matrix.example }}
- name: Verify example
if: matrix.verify
run: ${{ matrix.verify }}
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ fmt: ## Format and simplify the source code using `gofmt`

.PHONY: generate
generate: \
axiom/query/result_string.go \
axiom/query/aggregation_string.go \
axiom/querylegacy/aggregation_string.go \
axiom/querylegacy/filter_string.go \
axiom/querylegacy/kind_string.go \
Expand Down
18 changes: 12 additions & 6 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
[![Latest Release][release_badge]][release]
[![License][license_badge]][license]


[Axiom](https://axiom.co) unlocks observability at any scale.

- **Ingest with ease, store without limits:** Axiom's next-generation datastore
Expand Down Expand Up @@ -93,22 +92,29 @@ func main() {

client, err := axiom.NewClient()
if err != nil {
log.Fatalln(err)
log.Fatal(err)
}

if _, err = client.IngestEvents(ctx, "my-dataset", []axiom.Event{
{ingest.TimestampField: time.Now(), "foo": "bar"},
{ingest.TimestampField: time.Now(), "bar": "foo"},
}); err != nil {
log.Fatalln(err)
log.Fatal(err)
}

res, err := client.Query(ctx, "['my-dataset'] | where foo == 'bar' | limit 100")
if err != nil {
log.Fatalln(err)
log.Fatal(err)
} else if res.Status.RowsMatched == 0 {
log.Fatal("No matches found")
}
for _, match := range res.Matches {
fmt.Println(match.Data)

rows := res.Tables[0].Rows()
if err := rows.Range(ctx, func(_ context.Context, row query.Row) error {
_, err := fmt.Println(row)
return err
}); err != nil {
log.Fatal(err)
}
}
```
Expand Down
110 changes: 35 additions & 75 deletions axiom/datasets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,42 +124,10 @@ type aplQueryRequest struct {
type aplQueryResponse struct {
query.Result

// HINT(lukasmalkmus): Ignore these fields as they are not relevant for the
// user and/or will change with the new query result format.
LegacyRequest struct {
StartTime any `json:"startTime"`
EndTime any `json:"endTime"`
Resolution any `json:"resolution"`
Aggregations any `json:"aggregations"`
Filter any `json:"filter"`
Order any `json:"order"`
Limit any `json:"limit"`
VirtualFields any `json:"virtualFields"`
Projections any `json:"project"`
Cursor any `json:"cursor"`
IncludeCursor any `json:"includeCursor"`
ContinuationToken any `json:"continuationToken"`

// HINT(lukasmalkmus): Preserve the legacy request's "groupBy"
// field for now. This is needed to properly render some results.
GroupBy []string `json:"groupBy"`
} `json:"request"`
FieldsMeta any `json:"fieldsMetaMap"`
}

// UnmarshalJSON implements [json.Unmarshaler]. It is in place to unmarshal the
// groupBy field of the legacy request that is part of the response into the
// actual [query.Result.GroupBy] field.
func (r *aplQueryResponse) UnmarshalJSON(b []byte) error {
type localResponse *aplQueryResponse

if err := json.Unmarshal(b, localResponse(r)); err != nil {
return err
}

r.GroupBy = r.LegacyRequest.GroupBy

return nil
Format any `json:"format"`
Request any `json:"request"`
DatasetNames any `json:"datasetNames"`
FieldsMetaMap any `json:"fieldsMetaMap"`
}

// DatasetsService handles communication with the dataset related operations of
Expand Down Expand Up @@ -362,7 +330,7 @@ func (s *DatasetsService) Ingest(ctx context.Context, id string, r io.Reader, ty
}
res.TraceID = resp.TraceID()

setIngestResultOnSpan(span, res)
setIngestStatusOnSpan(span, res)

return &res, nil
}
Expand Down Expand Up @@ -468,7 +436,7 @@ func (s *DatasetsService) IngestEvents(ctx context.Context, id string, events []
}
res.TraceID = resp.TraceID()

setIngestResultOnSpan(span, res)
setIngestStatusOnSpan(span, res)

return &res, nil
}
Expand Down Expand Up @@ -525,7 +493,7 @@ func (s *DatasetsService) IngestChannel(ctx context.Context, id string, events <

var ingestStatus ingest.Status
defer func() {
setIngestResultOnSpan(span, ingestStatus)
setIngestStatusOnSpan(span, ingestStatus)
}()

flush := func() error {
Expand Down Expand Up @@ -598,7 +566,7 @@ func (s *DatasetsService) Query(ctx context.Context, apl string, options ...quer
queryParams := struct {
Format string `url:"format"`
}{
Format: "legacy", // Hardcode legacy APL format for now.
Format: "tabular", // Hardcode tabular result format for now.
}

// TODO(lukasmalkmus): Use 's.basePath' once ingest v2 is available.
Expand Down Expand Up @@ -627,7 +595,8 @@ func (s *DatasetsService) Query(ctx context.Context, apl string, options ...quer
}
res.TraceID = resp.TraceID()

setQueryResultOnSpan(span, res.Result)
setQueryStatusOnSpan(span, res.Result.Status)
span.SetAttributes(attribute.String("axiom.trace_id", res.TraceID))

return &res.Result, nil
}
Expand Down Expand Up @@ -679,7 +648,8 @@ func (s *DatasetsService) QueryLegacy(ctx context.Context, id string, q queryleg
res.SavedQueryID = resp.Header.Get("X-Axiom-History-Query-Id")
res.TraceID = resp.TraceID()

setLegacyQueryResultOnSpan(span, res.Result)
setLegacyQueryStatusOnSpan(span, res.Result.Status)
span.SetAttributes(attribute.String("axiom.trace_id", res.TraceID))

return &res.Result, nil
}
Expand Down Expand Up @@ -731,60 +701,50 @@ func DetectContentType(r io.Reader) (io.Reader, ContentType, error) {
return r, typ, nil
}

func setIngestResultOnSpan(span trace.Span, res ingest.Status) {
func setIngestStatusOnSpan(span trace.Span, status ingest.Status) {
if !span.IsRecording() {
return
}

span.SetAttributes(
attribute.String("axiom.result.trace_id", res.TraceID),
attribute.Int64("axiom.events.ingested", int64(res.Ingested)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.events.failed", int64(res.Failed)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.events.processed_bytes", int64(res.ProcessedBytes)), //nolint:gosec // Fine for this use case.
attribute.String("axiom.trace_id", status.TraceID),
attribute.Int64("axiom.events.ingested", int64(status.Ingested)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.events.failed", int64(status.Failed)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.events.processed_bytes", int64(status.ProcessedBytes)), //nolint:gosec // Fine for this use case.
)
}

//nolint:dupl // We need to support both query packages and their types.
func setQueryResultOnSpan(span trace.Span, res query.Result) {
func setQueryStatusOnSpan(span trace.Span, status query.Status) {
if !span.IsRecording() {
return
}

span.SetAttributes(
attribute.String("axiom.result.trace_id", res.TraceID),
attribute.String("axiom.result.status.elapsed_time", res.Status.ElapsedTime.String()),
attribute.Int64("axiom.result.status.blocks_examined", int64(res.Status.BlocksExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.result.status.rows_examined", int64(res.Status.RowsExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.result.status.rows_matched", int64(res.Status.RowsMatched)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.result.status.num_groups", int64(res.Status.NumGroups)),
attribute.Bool("axiom.result.status.is_partial", res.Status.IsPartial),
attribute.Bool("axiom.result.status.is_estimate", res.Status.IsEstimate),
attribute.String("axiom.result.status.min_block_time", res.Status.MinBlockTime.String()),
attribute.String("axiom.result.status.max_block_time", res.Status.MaxBlockTime.String()),
attribute.String("axiom.result.status.min_cursor", res.Status.MinCursor),
attribute.String("axiom.result.status.max_cursor", res.Status.MaxCursor),
attribute.String("axiom.query.min_cursor", status.MinCursor),
attribute.String("axiom.query.max_cursor", status.MaxCursor),
attribute.String("axiom.query.elapsed_time", status.ElapsedTime.String()),
attribute.Int64("axiom.query.rows_examined", int64(status.RowsExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.query.rows_matched", int64(status.RowsMatched)), //nolint:gosec // Fine for this use case.
)
}

//nolint:dupl // We need to support both query packages and their types.
func setLegacyQueryResultOnSpan(span trace.Span, res querylegacy.Result) {
func setLegacyQueryStatusOnSpan(span trace.Span, status querylegacy.Status) {
if !span.IsRecording() {
return
}

span.SetAttributes(
attribute.String("axiom.result.trace_id", res.TraceID),
attribute.String("axiom.result.status.elapsed_time", res.Status.ElapsedTime.String()),
attribute.Int64("axiom.result.status.blocks_examined", int64(res.Status.BlocksExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.result.status.rows_examined", int64(res.Status.RowsExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.result.status.rows_matched", int64(res.Status.RowsMatched)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.result.status.num_groups", int64(res.Status.NumGroups)),
attribute.Bool("axiom.result.status.is_partial", res.Status.IsPartial),
attribute.Bool("axiom.result.status.is_estimate", res.Status.IsEstimate),
attribute.String("axiom.result.status.min_block_time", res.Status.MinBlockTime.String()),
attribute.String("axiom.result.status.max_block_time", res.Status.MaxBlockTime.String()),
attribute.String("axiom.result.status.min_cursor", res.Status.MinCursor),
attribute.String("axiom.result.status.max_cursor", res.Status.MaxCursor),
attribute.String("axiom.querylegacy.elapsed_time", status.ElapsedTime.String()),
attribute.Int64("axiom.querylegacy.blocks_examined", int64(status.BlocksExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.querylegacy.rows_examined", int64(status.RowsExamined)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.querylegacy.rows_matched", int64(status.RowsMatched)), //nolint:gosec // Fine for this use case.
attribute.Int64("axiom.querylegacy.num_groups", int64(status.NumGroups)),
attribute.Bool("axiom.querylegacy.is_partial", status.IsPartial),
attribute.Bool("axiom.querylegacy.is_estimate", status.IsEstimate),
attribute.String("axiom.querylegacy.min_block_time", status.MinBlockTime.String()),
attribute.String("axiom.querylegacy.max_block_time", status.MaxBlockTime.String()),
attribute.String("axiom.querylegacy.min_cursor", status.MinCursor),
attribute.String("axiom.querylegacy.max_cursor", status.MaxCursor),
)
}

Expand Down
Loading
Loading