Skip to content

Commit

Permalink
feat: shared access support
Browse files Browse the repository at this point in the history
  • Loading branch information
lukasmalkmus committed Oct 11, 2023
1 parent 892cc2e commit bc65545
Show file tree
Hide file tree
Showing 27 changed files with 783 additions and 46 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ jobs:
AXIOM_URL: ${{ secrets[format('TESTING_{0}_API_URL', matrix.slug)] }}
AXIOM_TOKEN: ${{ secrets[format('TESTING_{0}_TOKEN', matrix.slug)] }}
AXIOM_ORG_ID: ${{ secrets[format('TESTING_{0}_ORG_ID', matrix.slug)] }}
AXIOM_SIGNING_KEY: ${{ secrets[format('TESTING_{0}_SHARED_ACCESS_SIGNING_KEY', matrix.slug)] }}
AXIOM_DATASET_SUFFIX: ${{ github.run_id }}-${{ matrix.go }}
TELEMETRY_TRACES_URL: ${{ secrets.TELEMETRY_TRACES_URL }}
TELEMETRY_TRACES_TOKEN: ${{ secrets.TELEMETRY_TRACES_TOKEN }}
Expand All @@ -126,7 +127,7 @@ jobs:
- name: Cleanup (On Test Failure)
if: failure()
run: |
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.10.0 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.11.3 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
axiom dataset list -f=json | jq '.[] | select(.id | contains("${{ github.run_id }}-${{ matrix.go }}")).id' | xargs -r -n1 axiom dataset delete -f
ci-pass:
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/push.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ jobs:
AXIOM_URL: ${{ secrets[format('TESTING_{0}_API_URL', matrix.slug)] }}
AXIOM_TOKEN: ${{ secrets[format('TESTING_{0}_TOKEN', matrix.slug)] }}
AXIOM_ORG_ID: ${{ secrets[format('TESTING_{0}_ORG_ID', matrix.slug)] }}
AXIOM_SIGNING_KEY: ${{ secrets[format('TESTING_{0}_SHARED_ACCESS_SIGNING_KEY', matrix.slug)] }}
AXIOM_DATASET_SUFFIX: ${{ github.run_id }}-${{ matrix.go }}
TELEMETRY_TRACES_URL: ${{ secrets.TELEMETRY_TRACES_URL }}
TELEMETRY_TRACES_TOKEN: ${{ secrets.TELEMETRY_TRACES_TOKEN }}
Expand All @@ -91,5 +92,5 @@ jobs:
- name: Cleanup (On Test Failure)
if: failure()
run: |
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.10.0 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.11.3 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
axiom dataset list -f=json | jq '.[] | select(.id | contains("${{ github.run_id }}-${{ matrix.go }}")).id' | xargs -r -n1 axiom dataset delete -f
3 changes: 2 additions & 1 deletion .github/workflows/server_regression.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
AXIOM_URL: ${{ secrets[format('TESTING_{0}_API_URL', matrix.slug)] }}
AXIOM_TOKEN: ${{ secrets[format('TESTING_{0}_TOKEN', matrix.slug)] }}
AXIOM_ORG_ID: ${{ secrets[format('TESTING_{0}_ORG_ID', matrix.slug)] }}
AXIOM_SIGNING_KEY: ${{ secrets[format('TESTING_{0}_SHARED_ACCESS_SIGNING_KEY', matrix.slug)] }}
AXIOM_DATASET_SUFFIX: ${{ github.run_id }}-${{ matrix.go }}
TELEMETRY_TRACES_URL: ${{ secrets.TELEMETRY_TRACES_URL }}
TELEMETRY_TRACES_TOKEN: ${{ secrets.TELEMETRY_TRACES_TOKEN }}
Expand All @@ -51,5 +52,5 @@ jobs:
- name: Cleanup (On Test Failure)
if: failure()
run: |
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.10.0 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.11.3 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
axiom dataset list -f=json | jq '.[] | select(.id | contains("${{ github.run_id }}-${{ matrix.go }}")).id' | xargs -r -n1 axiom dataset delete -f
4 changes: 3 additions & 1 deletion .github/workflows/test_examples.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ jobs:
- oteltraces
- query
- querylegacy
- sas
# HINT(lukasmalkmus): This test would require Go 1.21 (but uses Go
# 1.20 as specified in go.mod) but for the sake of simplicity we do
# not test it and trust the slogx test which is the same
Expand Down Expand Up @@ -92,6 +93,7 @@ jobs:
AXIOM_URL: ${{ secrets.TESTING_STAGING_API_URL }}
AXIOM_TOKEN: ${{ secrets.TESTING_STAGING_TOKEN }}
AXIOM_ORG_ID: ${{ secrets.TESTING_STAGING_ORG_ID }}
AXIOM_SIGNING_KEY: ${{ secrets.TESTING_STAGING_SHARED_ACCESS_SIGNING_KEY }}
AXIOM_DATASET: test-axiom-go-examples-${{ github.run_id }}-${{ matrix.example }}
steps:
- uses: actions/checkout@v4
Expand All @@ -100,7 +102,7 @@ jobs:
go-version-file: go.mod
- name: Setup test dataset
run: |
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.10.0 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
curl -sL $(curl -s https://api.github.com/repos/axiomhq/cli/releases/tags/v0.11.3 | grep "http.*linux_amd64.tar.gz" | awk '{print $2}' | sed 's|[\"\,]*||g') | tar xzvf - --strip-components=1 --wildcards -C /usr/local/bin "axiom_*_linux_amd64/axiom"
axiom dataset create -n=$AXIOM_DATASET -d="Axiom Go ${{ matrix.example }} example test"
- name: Setup example
if: matrix.setup
Expand Down
5 changes: 1 addition & 4 deletions adapters/zap/zap_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,7 @@ func Test(t *testing.T) {
require.NoError(t, err)

logger := zap.New(core)
defer func() {
err := logger.Sync()
assert.NoError(t, err)
}()
defer func() { assert.NoError(t, logger.Sync()) }()

logger.Info("This is awesome!", zap.String("mood", "hyped"))
logger.Warn("This is no that awesome...", zap.String("mood", "worried"))
Expand Down
4 changes: 2 additions & 2 deletions axiom/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ const (
otelTracerName = "github.com/axiomhq/axiom-go/axiom"
)

var validOnlyAPITokenPaths = regexp.MustCompile(`^/v1/datasets/([^/]+/(ingest|query)|_apl)(\?.+)?$`)
var validAPITokenPaths = regexp.MustCompile(`^/v1/datasets/([^/]+/(ingest|query)|_apl)(\?.+)?$`)

// service is the base service used by all Axiom API services.
type service struct {
Expand Down Expand Up @@ -194,7 +194,7 @@ func (c *Client) NewRequest(ctx context.Context, method, path string, body any)
}
endpoint := c.config.BaseURL().ResolveReference(rel)

if config.IsAPIToken(c.config.Token()) && !validOnlyAPITokenPaths.MatchString(endpoint.Path) {
if config.IsAPIToken(c.config.Token()) && !validAPITokenPaths.MatchString(endpoint.Path) {
return nil, ErrUnprivilegedToken
}

Expand Down
2 changes: 1 addition & 1 deletion axiom/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -692,7 +692,7 @@ func TestAPITokenPathRegex(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
assert.Equal(t, tt.match, validOnlyAPITokenPaths.MatchString(tt.input))
assert.Equal(t, tt.match, validAPITokenPaths.MatchString(tt.input))
})
}
}
Expand Down
16 changes: 9 additions & 7 deletions axiom/datasets.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ func (s *DatasetsService) Delete(ctx context.Context, id string) error {
))
defer span.End()

path, err := url.JoinPath(s.basePath, "/", id)
path, err := url.JoinPath(s.basePath, id)
if err != nil {
return spanError(span, err)
}
Expand Down Expand Up @@ -533,9 +533,7 @@ func (s *DatasetsService) IngestChannel(ctx context.Context, id string, events <
defer t.Stop()

var ingestStatus ingest.Status
defer func() {
setIngestResultOnSpan(span, ingestStatus)
}()
defer func() { setIngestResultOnSpan(span, ingestStatus) }()

flush := func() error {
if len(batch) == 0 {
Expand Down Expand Up @@ -596,9 +594,10 @@ func (s *DatasetsService) Query(ctx context.Context, apl string, options ...quer

ctx, span := s.client.trace(ctx, "Datasets.Query", trace.WithAttributes(
attribute.String("axiom.param.apl", apl),
attribute.String("axiom.param.start_time", opts.StartTime.String()),
attribute.String("axiom.param.end_time", opts.EndTime.String()),
attribute.String("axiom.param.start_time", opts.StartTime),
attribute.String("axiom.param.end_time", opts.EndTime),
attribute.String("axiom.param.cursor", opts.Cursor),
attribute.Bool("axiom.param.include_cursor", opts.IncludeCursor),
))
defer span.End()

Expand Down Expand Up @@ -648,7 +647,10 @@ func (s *DatasetsService) Query(ctx context.Context, apl string, options ...quer
// the future. Use [DatasetsService.Query] instead.
func (s *DatasetsService) QueryLegacy(ctx context.Context, id string, q querylegacy.Query, opts querylegacy.Options) (*querylegacy.Result, error) {
ctx, span := s.client.trace(ctx, "Datasets.QueryLegacy", trace.WithAttributes(
attribute.String("axiom.dataset_id", id),
attribute.String("axiom.param.dataset_id", id),
attribute.String("axiom.param.streaming_duration", opts.StreamingDuration.String()),
attribute.Bool("axiom.param.no_cache", opts.NoCache),
attribute.String("axiom.param.save_kind", opts.SaveKind.String()),
))
defer span.End()

Expand Down
1 change: 1 addition & 0 deletions axiom/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
// import "github.com/axiomhq/axiom-go/axiom/otel" // When using OpenTelemetry
// import "github.com/axiomhq/axiom-go/axiom/query" // When constructing APL queries
// import "github.com/axiomhq/axiom-go/axiom/querylegacy" // When constructing legacy queries
// import "github.com/axiomhq/axiom-go/axiom/sas" // When using shared access
//
// Construct a new Axiom client, then use the various services on the client to
// access different parts of the Axiom API. The package automatically takes its
Expand Down
5 changes: 1 addition & 4 deletions axiom/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,7 @@ func TestGzipEncoder(t *testing.T) {

gzr, err := gzip.NewReader(r)
require.NoError(t, err)
defer func() {
closeErr := gzr.Close()
require.NoError(t, closeErr)
}()
defer func() { require.NoError(t, gzr.Close()) }()

act, err := io.ReadAll(gzr)
require.NoError(t, err)
Expand Down
4 changes: 2 additions & 2 deletions axiom/limit.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,12 @@ func limitScopeFromString(s string) (ls LimitScope, err error) {
type Limit struct {
// Scope a limit is enforced for. Only present on rate limited requests.
Scope LimitScope
// The maximum limit a client is limited to for a specified time window
// The maximum limit a client is limited to for a specified time range
// which resets at the time indicated by [Limit.Reset].
Limit uint64
// The remaining count towards the maximum limit.
Remaining uint64
// The time at which the current limit time window will reset.
// The time at which the current limit time range will reset.
Reset time.Time

limitType limitType
Expand Down
13 changes: 12 additions & 1 deletion axiom/orgs.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,17 @@ func (l *License) UnmarshalJSON(b []byte) error {
return nil
}

// SigningKeys are the signing keys used to sign shared access tokens that
// can be used by a third party to run queries on behalf of the organization.
// They can be rotated.
type SigningKeys struct {
// Primary signing key. Gets rotated to the secondary signing key after
// rotation.
Primary string `json:"primary"`
// Secondary signing key. Gets rotated out.
Secondary string `json:"secondary"`
}

// Organization represents an organization.
type Organization struct {
// ID is the unique ID of the organization.
Expand Down Expand Up @@ -256,7 +267,7 @@ func (s *OrganizationsService) List(ctx context.Context) ([]*Organization, error
// Get an organization by id.
func (s *OrganizationsService) Get(ctx context.Context, id string) (*Organization, error) {
ctx, span := s.client.trace(ctx, "Organizations.Get", trace.WithAttributes(
attribute.String("axiom.dataset_id", id),
attribute.String("axiom.organization_id", id),
))
defer span.End()

Expand Down
22 changes: 16 additions & 6 deletions axiom/query/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import "time"
// Options specifies the optional parameters for a query.
type Options struct {
// StartTime for the interval to query.
StartTime time.Time `json:"startTime,omitempty"`
StartTime string `json:"startTime,omitempty"`
// EndTime of the interval to query.
EndTime time.Time `json:"endTime,omitempty"`
EndTime string `json:"endTime,omitempty"`
// Cursor to use for pagination. When used, don't specify new start and end
// times but rather use the start and end times of the query that returned
// the cursor that will be used.
Expand All @@ -27,15 +27,15 @@ type Option func(*Options)
// SetStartTime specifies the start time of the query interval. When also using
// [SetCursor], please make sure to use the start time of the query that
// returned the cursor that will be used.
func SetStartTime(startTime time.Time) Option {
return func(o *Options) { o.StartTime = startTime }
func SetStartTime[T time.Time | string](startTime T) Option {
return func(o *Options) { o.StartTime = timeOrStringToString(startTime) }
}

// SetEndTime specifies the end time of the query interval. When also using
// [SetCursor], please make sure to use the end time of the query that returned
// the cursor that will be used.
func SetEndTime(endTime time.Time) Option {
return func(o *Options) { o.EndTime = endTime }
func SetEndTime[T time.Time | string](endTime T) Option {
return func(o *Options) { o.EndTime = timeOrStringToString(endTime) }
}

// SetCursor specifies the cursor of the query. If include is set to true the
Expand Down Expand Up @@ -65,3 +65,13 @@ func SetVariable(name string, value any) Option {
func SetVariables(variables map[string]any) Option {
return func(o *Options) { o.Variables = variables }
}

func timeOrStringToString[T time.Time | string](t T) string {
switch t := any(t).(type) {
case time.Time:
return t.Format(time.RFC3339Nano)
case string:
return t
}
panic("time is neither time.Time nor string")
}
16 changes: 16 additions & 0 deletions axiom/sas/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Package sas implements functionality for creating and verifying shared access
// signatures (SAS) and shared access tokens (SAT) as well as using them to
// query Axiom datasets. A SAS grants querying capabilities to a dataset for a
// given time range and with a global filter applied on behalf of an
// organization. A SAS is an URL query string composed of a set of query
// parameters that make up the payload for a signature and the cryptographic
// signature itself. That cryptographic signature is called SAT.
//
// Usage:
//
// import "github.com/axiomhq/axiom-go/axiom/sas"
//
// To create a SAS, that can be attached to a query request, use the
// high-level [Create] function. The returned [Options] can be attached to a
// [http.Request] or encoded to a query string by calling [Options.Encode].
package sas
95 changes: 95 additions & 0 deletions axiom/sas/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package sas

import (
"errors"
"net/http"
"net/url"

"github.com/google/go-querystring/query"
)

// The parameter names for the shared access signature query string.
const (
queryOrgID = "oi"
queryDataset = "dt"
queryFilter = "fl"
queryMinStartTime = "mst"
queryMaxEndTime = "met"
queryExpiryTime = "exp"
queryToken = "tk"
)

// Options are the url query parameters used to authenticate a query request.
type Options struct {
Params

// Token is the signature created from the other fields in the options.
Token string `url:"tk"`
}

// Decode decodes the given signature into a set of options.
func Decode(signature string) (Options, error) {
q, err := url.ParseQuery(signature)
if err != nil {
return Options{}, err
}

options := Options{
Params: Params{
OrganizationID: q.Get(queryOrgID),
Dataset: q.Get(queryDataset),
Filter: q.Get(queryFilter),
MinStartTime: q.Get(queryMinStartTime),
MaxEndTime: q.Get(queryMaxEndTime),
ExpiryTime: q.Get(queryExpiryTime),
},
Token: q.Get(queryToken),
}

// Validate that the params are valid and the token is present.
if err := options.Params.Validate(); err != nil {
return options, err
} else if options.Token == "" {
return options, errors.New("missing token")
}

return options, nil
}

// Attach attaches the options to the given request as a query string. Existing
// query parameters are retained unless they are overwritten by the key of one
// of the options.
func (o Options) Attach(req *http.Request) error {
q, err := query.Values(o)
if err != nil {
return err
}

qc := req.URL.Query()
for k := range q {
qc.Set(k, q.Get(k))
}
req.URL.RawQuery = qc.Encode()

return nil
}

// Encode encodes the options into a url query string.
func (o Options) Encode() (string, error) {
q, err := query.Values(o)
if err != nil {
return "", err
}

// Although officially there is no limit specified by RFC 2616, many
// security protocols and recommendations state that maxQueryStrings on a
// server should be set to a maximum character limit of 1024. While the
// entire URL, including the querystring, should be set to a max of 2048
// characters.
s := q.Encode()
if len(s) > 1023 { // 1024 - 1 for '?'
return "", errors.New("signature too long")
}

return s, nil
}
Loading

0 comments on commit bc65545

Please sign in to comment.