From 3b42cb7b9cdeb40bda4281a0e9d565c0acf0a3d8 Mon Sep 17 00:00:00 2001 From: Lukas Malkmus Date: Tue, 18 Jul 2023 10:21:23 +0200 Subject: [PATCH] feat(adapters): slog --- .github/workflows/test_examples.yaml | 4 + adapters/README.md | 15 +- adapters/slog/README.md | 50 ++++++ adapters/slog/doc.go | 3 + adapters/slog/slog.go | 219 +++++++++++++++++++++++++ adapters/slog/slog_example_test.go | 25 +++ adapters/slog/slog_integration_test.go | 33 ++++ adapters/slog/slog_test.go | 118 +++++++++++++ examples/README.md | 2 + examples/slog/main.go | 37 +++++ go.mod | 2 +- go.sum | 4 +- 12 files changed, 507 insertions(+), 5 deletions(-) create mode 100644 adapters/slog/README.md create mode 100644 adapters/slog/doc.go create mode 100644 adapters/slog/slog.go create mode 100644 adapters/slog/slog_example_test.go create mode 100644 adapters/slog/slog_integration_test.go create mode 100644 adapters/slog/slog_test.go create mode 100644 examples/slog/main.go diff --git a/.github/workflows/test_examples.yaml b/.github/workflows/test_examples.yaml index d121af27..a2ba79bb 100644 --- a/.github/workflows/test_examples.yaml +++ b/.github/workflows/test_examples.yaml @@ -39,6 +39,7 @@ jobs: - oteltraces - query - querylegacy + - slog - zap include: - example: apex @@ -71,6 +72,9 @@ jobs: echo '[{"mood":"hyped","msg":"This is awesome!"}]' >> logs.json axiom ingest $AXIOM_DATASET -f=logs.json -f=logs.json -f=logs.json sleep 5 + - example: slog + verify: | + axiom dataset info $AXIOM_DATASET -f=json | jq -e 'any( .numEvents ; . == 3 )' - example: zap verify: | axiom dataset info $AXIOM_DATASET -f=json | jq -e 'any( .numEvents ; . == 3 )' diff --git a/adapters/README.md b/adapters/README.md index 75f28605..999fa29f 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -1,7 +1,18 @@ # Adapters -Adapters integrate Axiom Go into well known Go logging libraries. We currently -support these adapters right out of the box: +Adapters integrate Axiom Go into well known Go logging libraries. + +💡 _Go **1.21** will feature the `slog` package, a structured logging package in +the Go standard library. You can try it out know by importing +`golang.org/x/exp/slog` and we already provide [an adapter](slog)._ + +We currently support a bunch of adapters right out of the box. + +## Standard Library + +* [Slog](https://pkg.go.dev/golang.org/x/exp/slog): `import "github.com/axiomhq/axiom-go/adapters/slog"` + +## Third Party Packages * [Apex](https://github.com/apex/log): `import "github.com/axiomhq/axiom-go/adapters/apex"` * [Logrus](https://github.com/sirupsen/logrus): `import "github.com/axiomhq/axiom-go/adapters/logrus"` diff --git a/adapters/slog/README.md b/adapters/slog/README.md new file mode 100644 index 00000000..e0512153 --- /dev/null +++ b/adapters/slog/README.md @@ -0,0 +1,50 @@ +# Axiom Go Adapter for slog + +Adapter to ship logs generated by +[slog](https://pkg.go.dev/golang.org/x/exp/slog) to Axiom. + +## Quickstart + +Follow the [Axiom Go Quickstart](https://github.com/axiomhq/axiom-go#quickstart) +to install the Axiom Go package and configure your environment. + +Import the package: + +```go +// Imported as "adapter" to not conflict with the "slog" package. +import adapter "github.com/axiomhq/axiom-go/adapters/slog" +``` + +You can also configure the adapter using [options](https://pkg.go.dev/github.com/axiomhq/axiom-go/adapters/slog#Option) +passed to the [New](https://pkg.go.dev/github.com/axiomhq/axiom-go/adapters/slog#New) +function: + +```go +handler, err := adapter.New( + SetDataset("AXIOM_DATASET"), +) +``` + +To configure the underlying client manually either pass in a client that was +created according to the [Axiom Go Quickstart](https://github.com/axiomhq/axiom-go#quickstart) +using [SetClient](https://pkg.go.dev/github.com/axiomhq/axiom-go/adapters/slog#SetClient) +or pass [client options](https://pkg.go.dev/github.com/axiomhq/axiom-go/axiom#Option) +to the adapter using [SetClientOptions](https://pkg.go.dev/github.com/axiomhq/axiom-go/adapters/slog#SetClientOptions). + +```go +import adapter "github.com/axiomhq/axiom-go/axiom" + +// ... + +handler, err := adapter.New( + SetClientOptions( + axiom.SetPersonalTokenConfig("AXIOM_TOKEN", "AXIOM_ORG_ID"), + ), +) +``` + +### ❗ Important ❗ + +The adapter uses a buffer to batch events before sending them to Axiom. This +buffer must be flushed explicitly by calling [Close](https://pkg.go.dev/github.com/axiomhq/axiom-go/adapters/slog#Handler.Close). +Checkout the [example](../../examples/slog/main.go). diff --git a/adapters/slog/doc.go b/adapters/slog/doc.go new file mode 100644 index 00000000..385e2750 --- /dev/null +++ b/adapters/slog/doc.go @@ -0,0 +1,3 @@ +// Package slog provides an adapter for the standard libraries structured +// logging package. +package slog diff --git a/adapters/slog/slog.go b/adapters/slog/slog.go new file mode 100644 index 00000000..0d836704 --- /dev/null +++ b/adapters/slog/slog.go @@ -0,0 +1,219 @@ +package slog + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "os" + "sync" + "time" + + "golang.org/x/exp/slog" + + "github.com/axiomhq/axiom-go/axiom" + "github.com/axiomhq/axiom-go/axiom/ingest" +) + +var _ slog.Handler = (*Handler)(nil) + +var logger = log.New(os.Stderr, "[AXIOM|SLOG]", 0) + +// ErrMissingDatasetName is raised when a dataset name is not provided. Set it +// manually using the [SetDataset] option or export "AXIOM_DATASET". +var ErrMissingDatasetName = errors.New("missing dataset name") + +// An Option modifies the behaviour of the Axiom handler. +type Option func(*Handler) error + +// SetClient specifies the Axiom client to use for ingesting the logs. +func SetClient(client *axiom.Client) Option { + return func(h *Handler) error { + h.client = client + return nil + } +} + +// SetClientOptions specifies the Axiom client options to pass to +// [axiom.NewClient] which is only called if no [axiom.Client] was specified by +// the [SetClient] option. +func SetClientOptions(options ...axiom.Option) Option { + return func(h *Handler) error { + h.clientOptions = options + return nil + } +} + +// SetDataset specifies the dataset to ingest the logs into. Can also be +// specified using the "AXIOM_DATASET" environment variable. +func SetDataset(datasetName string) Option { + return func(h *Handler) error { + h.datasetName = datasetName + return nil + } +} + +// SetHandlerOptions specifies the handler options to use. +func SetHandlerOptions(opts *slog.HandlerOptions) Option { + return func(h *Handler) error { + h.handlerOptions = opts + return nil + } +} + +// SetIngestOptions specifies the ingestion options to use for ingesting the +// logs. +func SetIngestOptions(opts ...ingest.Option) Option { + return func(h *Handler) error { + h.ingestOptions = opts + return nil + } +} + +// Handler implements a [slog.Handler] used for shipping logs to Axiom. +type Handler struct { + client *axiom.Client + datasetName string + + clientOptions []axiom.Option + handlerOptions *slog.HandlerOptions + ingestOptions []ingest.Option + + jsonHandler *slog.JSONHandler + buf bytes.Buffer + bufMtx sync.Mutex + closeCh chan struct{} + closeOnce sync.Once +} + +// New creates a new handler that ingests logs into Axiom. It automatically +// takes its configuration from the environment. To connect, export the +// following environment variables: +// +// - AXIOM_TOKEN +// - AXIOM_ORG_ID (only when using a personal token) +// - AXIOM_DATASET +// +// The configuration can be set manually using options which are prefixed with +// "Set". +// +// An api token with "ingest" permission is sufficient enough. +// +// A handler needs to be closed properly to make sure all logs are sent by +// calling [Handler.Close]. +func New(options ...Option) (*Handler, error) { + handler := &Handler{ + closeCh: make(chan struct{}), + } + + // Apply supplied options. + for _, option := range options { + if err := option(handler); err != nil { + return nil, err + } + } + + // Create client, if not set. + if handler.client == nil { + var err error + if handler.client, err = axiom.NewClient(handler.clientOptions...); err != nil { + return nil, err + } + } + + // When the dataset name is not set, use "AXIOM_DATASET". + if handler.datasetName == "" { + handler.datasetName = os.Getenv("AXIOM_DATASET") + if handler.datasetName == "" { + return nil, ErrMissingDatasetName + } + } + + // Create a JSON handler. + handler.jsonHandler = slog.NewJSONHandler(&handler.buf, handler.handlerOptions) + + // Run background ingest. + go func() { + if err := handler.sync(context.Background()); err != nil { + logger.Printf("failed to ingest events: %s\n", err) + } + }() + + return handler, nil +} + +// Close the handler and make sure all events are flushed. Closing the handler +// renders it unusable for further use. +func (h *Handler) Close() { + h.closeOnce.Do(func() { + close(h.closeCh) + }) +} + +// Enabled implements [slog.Handler]. +func (h *Handler) Enabled(ctx context.Context, level slog.Level) bool { + return h.jsonHandler.Enabled(ctx, level) +} + +// Handle implements [slog.Handler]. +func (h *Handler) Handle(ctx context.Context, r slog.Record) error { + return h.jsonHandler.Handle(ctx, r) +} + +// WithAttrs implements [slog.Handler]. +func (h *Handler) WithAttrs(attrs []slog.Attr) slog.Handler { + return h.jsonHandler.WithAttrs(attrs) +} + +// WithGroup implements [slog.Handler]. +func (h *Handler) WithGroup(name string) slog.Handler { + return h.jsonHandler.WithGroup(name) +} + +func (h *Handler) sync(ctx context.Context) error { + // Flush on a per second basis. + const flushInterval = time.Second + t := time.NewTicker(flushInterval) + defer t.Stop() + + flush := func() error { + h.bufMtx.Lock() + defer h.bufMtx.Unlock() + + if h.buf.Len() == 0 { + return nil + } + + r, err := axiom.ZstdEncoder()(&h.buf) + if err != nil { + return err + } + + res, err := h.client.Ingest(ctx, h.datasetName, r, axiom.NDJSON, axiom.Zstd, h.ingestOptions...) + if err != nil { + return fmt.Errorf("failed to ingest events: %w", err) + } else if res.Failed > 0 { + // Best effort on notifying the user about the ingest failure. + logger.Printf("event at %s failed to ingest: %s\n", + res.Failures[0].Timestamp, res.Failures[0].Error) + } + t.Reset(flushInterval) // Reset the ticker. + h.buf.Reset() // Clear the buffer. + + return nil + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-h.closeCh: + return flush() + case <-t.C: + if err := flush(); err != nil { + return err + } + } + } +} diff --git a/adapters/slog/slog_example_test.go b/adapters/slog/slog_example_test.go new file mode 100644 index 00000000..68f87d1e --- /dev/null +++ b/adapters/slog/slog_example_test.go @@ -0,0 +1,25 @@ +package slog_test + +import ( + "log" + + "golang.org/x/exp/slog" + + adapter "github.com/axiomhq/axiom-go/adapters/slog" +) + +func Example() { + // Export "AXIOM_DATASET" in addition to the required environment variables. + + handler, err := adapter.New() + if err != nil { + log.Fatal(err.Error()) + } + defer handler.Close() + + logger := slog.New(handler) + + logger.Info("This is awesome!", "mood", "hyped") + logger.With("mood", "worried").Warn("This is no that awesome...") + logger.Error("This is rather bad.", slog.String("mood", "depressed")) +} diff --git a/adapters/slog/slog_integration_test.go b/adapters/slog/slog_integration_test.go new file mode 100644 index 00000000..e1628a82 --- /dev/null +++ b/adapters/slog/slog_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package slog_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/exp/slog" + + adapter "github.com/axiomhq/axiom-go/adapters/slog" + "github.com/axiomhq/axiom-go/axiom" + "github.com/axiomhq/axiom-go/internal/test/adapters" +) + +func Test(t *testing.T) { + adapters.IntegrationTest(t, "slog", func(_ context.Context, dataset string, client *axiom.Client) { + handler, err := adapter.New( + adapter.SetClient(client), + adapter.SetDataset(dataset), + ) + require.NoError(t, err) + + defer handler.Close() + + logger := slog.New(handler) + + logger.Info("This is awesome!", slog.String("mood", "hyped")) + logger.Warn("This is no that awesome...", slog.String("mood", "worried")) + logger.Error("This is rather bad.", slog.String("mood", "depressed")) + }) +} diff --git a/adapters/slog/slog_test.go b/adapters/slog/slog_test.go new file mode 100644 index 00000000..8a5b6b69 --- /dev/null +++ b/adapters/slog/slog_test.go @@ -0,0 +1,118 @@ +package slog + +import ( + "bufio" + "io" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/slog" + + "github.com/axiomhq/axiom-go/axiom" + "github.com/axiomhq/axiom-go/internal/test/adapters" + "github.com/axiomhq/axiom-go/internal/test/testhelper" +) + +// TestNew makes sure New() picks up the "AXIOM_DATASET" environment variable. +func TestNew(t *testing.T) { + testhelper.SafeClearEnv(t) + + t.Setenv("AXIOM_TOKEN", "xaat-test") + t.Setenv("AXIOM_ORG_ID", "123") + + handler, err := New() + require.ErrorIs(t, err, ErrMissingDatasetName) + require.Nil(t, handler) + + t.Setenv("AXIOM_DATASET", "test") + + handler, err = New() + require.NoError(t, err) + require.NotNil(t, handler) + + assert.Equal(t, "test", handler.datasetName) +} + +func TestHandler(t *testing.T) { + exp := `{"level":"INFO","key":"value","msg":"my message"}` + + var hasRun uint64 + hf := func(w http.ResponseWriter, r *http.Request) { + zsr, err := zstd.NewReader(r.Body) + require.NoError(t, err) + + b, err := io.ReadAll(zsr) + require.NoError(t, err) + + testhelper.JSONEqExp(t, exp, string(b), []string{"time"}) + + atomic.AddUint64(&hasRun, 1) + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("{}")) + } + + logger := adapters.Setup(t, hf, setup(t)) + + logger.Info("my message", "key", "value") + + // Wait for timer based handler flush. + time.Sleep(1250 * time.Millisecond) + + assert.EqualValues(t, 1, atomic.LoadUint64(&hasRun)) +} + +func TestHandler_FlushBatch(t *testing.T) { + var lines uint64 + hf := func(w http.ResponseWriter, r *http.Request) { + zsr, err := zstd.NewReader(r.Body) + require.NoError(t, err) + + s := bufio.NewScanner(zsr) + for s.Scan() { + atomic.AddUint64(&lines, 1) + } + assert.NoError(t, s.Err()) + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("{}")) + } + + logger := adapters.Setup(t, hf, setup(t)) + + for i := 0; i < 1000; i++ { + logger.Info("my message") + } + + // Wait just enough to not get a timer flush. + time.Sleep(250 * time.Millisecond) + + // Should not have any events. + assert.Zero(t, atomic.LoadUint64(&lines)) + + // Wait for timer based handler flush. + time.Sleep(1250 * time.Millisecond) + + // Should have received the full batch. + assert.EqualValues(t, 1000, atomic.LoadUint64(&lines)) +} + +func setup(t *testing.T) func(dataset string, client *axiom.Client) *slog.Logger { + return func(dataset string, client *axiom.Client) *slog.Logger { + t.Helper() + + handler, err := New( + SetClient(client), + SetDataset(dataset), + ) + require.NoError(t, err) + t.Cleanup(handler.Close) + + return slog.New(handler) + } +} diff --git a/examples/README.md b/examples/README.md index a015223f..94323020 100644 --- a/examples/README.md +++ b/examples/README.md @@ -52,6 +52,8 @@ eval $(axiom config export -f) [Apex](https://github.com/apex/log) logging package. - [logrus](logrus/main.go): How to ship logs to Axiom using the popular [Logrus](https://github.com/sirupsen/logrus) logging package. +- [slog](slog/main.go): How to ship logs to Axiom using the standard libraries + [Slog](https://pkg.go.dev/golang.org/x/exp/slog) structured logging package. - [zap](zap/main.go): How to ship logs to Axiom using the popular [Zap](https://github.com/uber-go/zap) logging package. diff --git a/examples/slog/main.go b/examples/slog/main.go new file mode 100644 index 00000000..fd544a2c --- /dev/null +++ b/examples/slog/main.go @@ -0,0 +1,37 @@ +// The purpose of this example is to show how to integrate with slog. +package main + +import ( + "log" + + "golang.org/x/exp/slog" + + adapter "github.com/axiomhq/axiom-go/adapters/slog" +) + +func main() { + // Export "AXIOM_DATASET" in addition to the required environment variables. + + // 1. Setup the Axiom handler for slog. + handler, err := adapter.New() + if err != nil { + log.Fatal(err.Error()) + } + + // 2. Have all logs flushed before the application exits. + // + // ❗THIS IS IMPORTANT❗ Without it, the logs will not be sent to Axiom as + // the buffer will not be flushed when the application exits. + defer handler.Close() + + // 3. Create the logger. + logger := slog.New(handler) + + // 4. 💡 Optional: Make the Go log package use the structured logger. + slog.SetDefault(logger) + + // 5. Log ⚡ + logger.Info("This is awesome!", "mood", "hyped") + logger.With("mood", "worried").Warn("This is no that awesome...") + logger.Error("This is rather bad.", slog.String("mood", "depressed")) +} diff --git a/go.mod b/go.mod index 1cb861ef..622da59a 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/sync v0.3.0 golang.org/x/tools v0.11.0 gotest.tools/gotestsum v1.10.1 @@ -198,7 +199,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.7.0 // indirect - golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.12.0 // indirect diff --git a/go.sum b/go.sum index fffec07d..8e36055d 100644 --- a/go.sum +++ b/go.sum @@ -710,8 +710,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 h1:J74nGeMgeFnYQJN59eFwh06jX/V8g0lB7LWpjSLxtgU=