From b0bb1f2f39aa227fc08506238af1719f568df2bb Mon Sep 17 00:00:00 2001 From: zepatrik Date: Fri, 18 Oct 2024 10:19:37 +0200 Subject: [PATCH] feat: write listen files with actual address --- embedx/config.schema.json | 28 +++++++++++ go.mod | 8 ++-- go.sum | 12 ++--- internal/driver/config/provider.go | 32 +++++++------ internal/driver/config/provider_test.go | 4 +- internal/driver/daemon.go | 37 ++++++++++++--- internal/driver/daemon_test.go | 42 ++++++++--------- internal/driver/testhelpers.go | 63 +++++++++++++++++++++++++ internal/e2e/full_suit_test.go | 45 ++++++++++-------- internal/e2e/grpc_client_test.go | 3 +- internal/e2e/helpers.go | 14 ++---- 11 files changed, 204 insertions(+), 84 deletions(-) create mode 100644 internal/driver/testhelpers.go diff --git a/embedx/config.schema.json b/embedx/config.schema.json index be38a0222..8f46f26a3 100644 --- a/embedx/config.schema.json +++ b/embedx/config.schema.json @@ -211,6 +211,13 @@ "title": "Host", "description": "The network interface to listen on." }, + "write_listen_file": { + "type": "string", + "title": "Read Listen File", + "description": "The path to a file that will be created when the read API is ready to accept connections. The content of the file is the host:port of the read API. Use this to get the actual port when using port 0. The service might not yet be ready to accept connections when the file is created.", + "format": "uri", + "examples": ["file:///tmp/keto-read-api"] + }, "cors": { "$ref": "#/definitions/cors" }, @@ -239,6 +246,13 @@ "title": "Host", "description": "The network interface to listen on." }, + "write_listen_file": { + "type": "string", + "title": "Write Listen File", + "description": "The path to a file that will be created when the write API is ready to accept connections. The content of the file is the host:port of the write API. Use this to get the actual port when using port 0. The service might not yet be ready to accept connections when the file is created.", + "format": "uri", + "examples": ["file:///tmp/keto-write-api"] + }, "cors": { "$ref": "#/definitions/cors" }, @@ -267,6 +281,13 @@ "title": "Host", "description": "The network interface to listen on." }, + "write_listen_file": { + "type": "string", + "title": "Metrics Listen File", + "description": "The path to a file that will be created when the metrics API is ready to accept connections. The content of the file is the host:port of the metrics API. Use this to get the actual port when using port 0. The service might not yet be ready to accept connections when the file is created.", + "format": "uri", + "examples": ["file:///tmp/keto-metrics-api"] + }, "cors": { "$ref": "#/definitions/cors" }, @@ -295,6 +316,13 @@ "title": "Host", "description": "The network interface to listen on." }, + "write_listen_file": { + "type": "string", + "title": "OPL Listen File", + "description": "The path to a file that will be created when the OPL API is ready to accept connections. The content of the file is the host:port of the OPL API. Use this to get the actual port when using port 0. The service might not yet be ready to accept connections when the file is created.", + "format": "uri", + "examples": ["file:///tmp/keto-opl-api"] + }, "cors": { "$ref": "#/definitions/cors" }, diff --git a/go.mod b/go.mod index 758e79207..05fc88062 100644 --- a/go.mod +++ b/go.mod @@ -22,9 +22,8 @@ require ( github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88 github.com/ory/jsonschema/v3 v3.0.8 github.com/ory/keto/proto v0.13.0-alpha.0 - github.com/ory/x v0.0.647 + github.com/ory/x v0.0.662 github.com/pelletier/go-toml v1.9.5 - github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.55.0 @@ -34,7 +33,7 @@ require ( github.com/soheilhy/cmux v0.1.5 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.9.1-0.20240613200951-b074924938f8 github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/urfave/negroni v1.0.0 @@ -146,11 +145,12 @@ require ( github.com/nyaruka/phonenumbers v1.1.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.1.13 // indirect + github.com/opencontainers/runc v1.1.14 // indirect github.com/openzipkin/zipkin-go v0.4.2 // indirect github.com/ory/dockertest/v3 v3.11.0 // indirect github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 // indirect github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.1 // indirect diff --git a/go.sum b/go.sum index 106fd492f..5a815e9b8 100644 --- a/go.sum +++ b/go.sum @@ -471,8 +471,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.1.13 h1:98S2srgG9vw0zWcDpFMn5TRrh8kLxa/5OFUstuUhmRs= -github.com/opencontainers/runc v1.1.13/go.mod h1:R016aXacfp/gwQBYw2FDGa9m+n6atbLWrYY8hNMT/sA= +github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= +github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA= github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hPpCJSBx/EYFhY= github.com/ory/analytics-go/v5 v5.0.1 h1:LX8T5B9FN8KZXOtxgN+R3I4THRRVB6+28IKgKBpXmAM= @@ -487,8 +487,8 @@ github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88 h1:J0CIFKdpUeqKbVMw github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88/go.mod h1:MMNmY6MG1uB6fnXYFaHoqdV23DTWctlPsmRCeq/2+wc= github.com/ory/jsonschema/v3 v3.0.8 h1:Ssdb3eJ4lDZ/+XnGkvQS/te0p+EkolqwTsDOCxr/FmU= github.com/ory/jsonschema/v3 v3.0.8/go.mod h1:ZPzqjDkwd3QTnb2Z6PAS+OTvBE2x5i6m25wCGx54W/0= -github.com/ory/x v0.0.647 h1:DVKgA3ykMB9qXuMdSl5C8SFWr3yw7Xe8jpSm0+iqGeU= -github.com/ory/x v0.0.647/go.mod h1:M+0EAXo7DT7Z2/Yrzvh4mgxOoV1fGI1jOKyAJ72d4Qs= +github.com/ory/x v0.0.662 h1:Qah5/f63Kr33Wcqm1S79adcYHaoWeiCDMQiDLuUXBqU= +github.com/ory/x v0.0.662/go.mod h1:tS0FyZXpVeKd1lCcFgV/Rb1GlccI/Xq8DraFS+lmIt8= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -587,8 +587,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.9.1-0.20240613200951-b074924938f8 h1:lMSOB3NNaQcNDK9eCjQb0LFfpXW03lzF2SN6vUziALo= +github.com/stretchr/testify v1.9.1-0.20240613200951-b074924938f8/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/internal/driver/config/provider.go b/internal/driver/config/provider.go index f9e1ca73a..bdeac5563 100644 --- a/internal/driver/config/provider.go +++ b/internal/driver/config/provider.go @@ -43,14 +43,18 @@ const ( KeyBatchCheckMaxBatchSize = "limit.max_batch_check_size" KeyBatchCheckParallelizationLimit = "limit.batch_check_max_parallelization" - KeyReadAPIHost = "serve." + string(EndpointRead) + ".host" - KeyReadAPIPort = "serve." + string(EndpointRead) + ".port" - KeyWriteAPIHost = "serve." + string(EndpointWrite) + ".host" - KeyWriteAPIPort = "serve." + string(EndpointWrite) + ".port" - KeyOPLSyntaxAPIHost = "serve." + string(EndpointOPLSyntax) + ".host" - KeyOPLSyntaxAPIPort = "serve." + string(EndpointOPLSyntax) + ".port" - KeyMetricsHost = "serve." + string(EndpointMetrics) + ".host" - KeyMetricsPort = "serve." + string(EndpointMetrics) + ".port" + KeyReadAPIHost = "serve." + string(EndpointRead) + ".host" + KeyReadAPIPort = "serve." + string(EndpointRead) + ".port" + KeyReadAPIListenFile = "serve." + string(EndpointRead) + ".write_listen_file" + KeyWriteAPIHost = "serve." + string(EndpointWrite) + ".host" + KeyWriteAPIPort = "serve." + string(EndpointWrite) + ".port" + KeyWriteAPIListenFile = "serve." + string(EndpointWrite) + ".write_listen_file" + KeyOPLSyntaxAPIHost = "serve." + string(EndpointOPLSyntax) + ".host" + KeyOPLSyntaxAPIPort = "serve." + string(EndpointOPLSyntax) + ".port" + KeyOPLSyntaxListenFile = "serve." + string(EndpointOPLSyntax) + ".write_listen_file" + KeyMetricsHost = "serve." + string(EndpointMetrics) + ".host" + KeyMetricsPort = "serve." + string(EndpointMetrics) + ".port" + KeyMetricsListenFile = "serve." + string(EndpointMetrics) + ".write_listen_file" KeyNamespaces = "namespaces" KeyNamespacesExperimentalStrictMode = KeyNamespaces + ".experimental_strict_mode" @@ -167,18 +171,18 @@ func (k *Config) Set(key string, v any) error { return nil } -func (k *Config) addressFor(endpoint EndpointType) string { +func (k *Config) addressFor(endpoint EndpointType) (string, string) { return fmt.Sprintf( "%s:%d", k.p.StringF("serve."+string(endpoint)+".host", ""), k.p.IntF("serve."+string(endpoint)+".port", 0), - ) + ), k.p.StringF("serve."+string(endpoint)+".write_listen_file", "") } -func (k *Config) ReadAPIListenOn() string { return k.addressFor(EndpointRead) } -func (k *Config) WriteAPIListenOn() string { return k.addressFor(EndpointWrite) } -func (k *Config) MetricsListenOn() string { return k.addressFor(EndpointMetrics) } -func (k *Config) OPLSyntaxAPIListenOn() string { return k.addressFor(EndpointOPLSyntax) } +func (k *Config) ReadAPIListenOn() (string, string) { return k.addressFor(EndpointRead) } +func (k *Config) WriteAPIListenOn() (string, string) { return k.addressFor(EndpointWrite) } +func (k *Config) MetricsListenOn() (string, string) { return k.addressFor(EndpointMetrics) } +func (k *Config) OPLSyntaxAPIListenOn() (string, string) { return k.addressFor(EndpointOPLSyntax) } func (k *Config) MaxReadDepth() int { return k.p.Int(KeyLimitMaxReadDepth) diff --git a/internal/driver/config/provider_test.go b/internal/driver/config/provider_test.go index 2628b2749..ffba74a55 100644 --- a/internal/driver/config/provider_test.go +++ b/internal/driver/config/provider_test.go @@ -283,5 +283,7 @@ func TestProvider_DefaultReadAPIListenOn(t *testing.T) { ) require.NoError(t, err) - assert.Equal(t, ":4466", config.ReadAPIListenOn()) + addr, listenFile := config.ReadAPIListenOn() + assert.Equal(t, ":4466", addr) + assert.Zero(t, listenFile) } diff --git a/internal/driver/daemon.go b/internal/driver/daemon.go index 88825e3df..f938b9558 100644 --- a/internal/driver/daemon.go +++ b/internal/driver/daemon.go @@ -5,6 +5,7 @@ package driver import ( "context" + "fmt" "net" "net/http" "os" @@ -163,7 +164,8 @@ func (r *RegistryDefault) serveRead(ctx context.Context, done chan<- struct{}) e rt = otelx.TraceHandler(rt, otelhttp.WithTracerProvider(tracer.Provider())) } - return multiplexPort(ctx, r.Logger().WithField("endpoint", "read"), r.Config(ctx).ReadAPIListenOn(), rt, s, done) + addr, listenFile := r.Config(ctx).ReadAPIListenOn() + return multiplexPort(ctx, r.Logger().WithField("endpoint", "read"), addr, listenFile, rt, s, done) } func (r *RegistryDefault) serveWrite(ctx context.Context, done chan<- struct{}) error { @@ -173,7 +175,8 @@ func (r *RegistryDefault) serveWrite(ctx context.Context, done chan<- struct{}) rt = otelx.TraceHandler(rt, otelhttp.WithTracerProvider(tracer.Provider())) } - return multiplexPort(ctx, r.Logger().WithField("endpoint", "write"), r.Config(ctx).WriteAPIListenOn(), rt, s, done) + addr, listenFile := r.Config(ctx).WriteAPIListenOn() + return multiplexPort(ctx, r.Logger().WithField("endpoint", "write"), addr, listenFile, rt, s, done) } func (r *RegistryDefault) serveOPLSyntax(ctx context.Context, done chan<- struct{}) error { @@ -183,23 +186,29 @@ func (r *RegistryDefault) serveOPLSyntax(ctx context.Context, done chan<- struct rt = otelx.TraceHandler(rt, otelhttp.WithTracerProvider(tracer.Provider())) } - return multiplexPort(ctx, r.Logger().WithField("endpoint", "opl"), r.Config(ctx).OPLSyntaxAPIListenOn(), rt, s, done) + addr, listenFile := r.Config(ctx).OPLSyntaxAPIListenOn() + return multiplexPort(ctx, r.Logger().WithField("endpoint", "opl"), addr, listenFile, rt, s, done) } func (r *RegistryDefault) serveMetrics(ctx context.Context, done chan<- struct{}) error { ctx, cancel := context.WithCancel(ctx) defer cancel() + addr, listenFile := r.Config(ctx).MetricsListenOn() + l, err := listenAndWriteFile(ctx, addr, listenFile) + if err != nil { + return err + } + //nolint:gosec // graceful.WithDefaults already sets a timeout s := graceful.WithDefaults(&http.Server{ Handler: r.metricsRouter(ctx), - Addr: r.Config(ctx).MetricsListenOn(), }) eg := &errgroup.Group{} eg.Go(func() error { - if err := s.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + if err := s.Serve(l); !errors.Is(err, http.ErrServerClosed) { return errors.WithStack(err) } return nil @@ -224,8 +233,8 @@ func (r *RegistryDefault) serveMetrics(ctx context.Context, done chan<- struct{} return eg.Wait() } -func multiplexPort(ctx context.Context, log *logrusx.Logger, addr string, router http.Handler, grpcS *grpc.Server, done chan<- struct{}) error { - l, err := (&net.ListenConfig{}).Listen(ctx, "tcp", addr) +func multiplexPort(ctx context.Context, log *logrusx.Logger, addr, listenFile string, router http.Handler, grpcS *grpc.Server, done chan<- struct{}) error { + l, err := listenAndWriteFile(ctx, addr, listenFile) if err != nil { return err } @@ -324,6 +333,20 @@ func (r *RegistryDefault) allHandlers() []Handler { return r.handlers } +func listenAndWriteFile(ctx context.Context, addr, listenFile string) (net.Listener, error) { + l, err := (&net.ListenConfig{}).Listen(ctx, "tcp", addr) + if err != nil { + return nil, errors.WithStack(fmt.Errorf("unable to listen on %q: %w", addr, err)) + } + const filePrefix = "file://" + if strings.HasPrefix(listenFile, filePrefix) { + if err := os.WriteFile(listenFile[len(filePrefix):], []byte(l.Addr().String()), 0600); err != nil { + return nil, errors.WithStack(fmt.Errorf("unable to write listen file %q: %w", listenFile, err)) + } + } + return l, nil +} + func (r *RegistryDefault) ReadRouter(ctx context.Context) http.Handler { n := negroni.New() for _, f := range r.defaultHttpMiddlewares { diff --git a/internal/driver/daemon_test.go b/internal/driver/daemon_test.go index fd5ccad56..e84fb3031 100644 --- a/internal/driver/daemon_test.go +++ b/internal/driver/daemon_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/phayes/freeport" "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/assert" "golang.org/x/sync/errgroup" @@ -19,8 +18,6 @@ import ( grpcHealthV1 "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/status" - "github.com/ory/keto/internal/driver/config" - "context" prometheus "github.com/ory/x/prometheusx" @@ -29,19 +26,13 @@ import ( ) func TestScrapingEndpoint(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - port, err := freeport.GetFreePort() - require.NoError(t, err) - r := NewSqliteTestRegistry(t, false) - require.NoError(t, r.Config(ctx).Set(config.KeyWriteAPIPort, port)) - - //metrics port - portMetrics, err := freeport.GetFreePort() - require.NoError(t, err) - require.NoError(t, r.Config(ctx).Set(config.KeyMetricsPort, portMetrics)) + getAddr := UseDynamicPorts(ctx, t, r) eg := errgroup.Group{} doneShutdown := make(chan struct{}) @@ -52,8 +43,13 @@ func TestScrapingEndpoint(t *testing.T) { return r.serveMetrics(ctx, doneShutdown) }) - require.EventuallyWithT(t, func(t *assert.CollectT) { - conn, err := grpc.DialContext(ctx, fmt.Sprintf("127.0.0.1:%d", port), grpc.WithTransportCredentials(insecure.NewCredentials())) + _, writePort, _ := getAddr(t, "write") + _, metricsPort, _ := getAddr(t, "metrics") + + t.Logf("write port: %s, metrics port: %s", writePort, metricsPort) + + assert.EventuallyWithT(t, func(t *assert.CollectT) { + conn, err := grpc.DialContext(ctx, fmt.Sprintf("127.0.0.1:%s", writePort), grpc.WithTransportCredentials(insecure.NewCredentials())) require.NoError(t, err) defer conn.Close() @@ -63,9 +59,9 @@ func TestScrapingEndpoint(t *testing.T) { require.NoError(t, watcher.CloseSend()) for err := status.Error(codes.Unavailable, "init"); status.Code(err) != codes.Unavailable; _, err = watcher.Recv() { } - }, 2*time.Second, 100*time.Millisecond) + }, 2*time.Second, 10*time.Millisecond) - promresp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d", portMetrics) + prometheus.MetricsPrometheusPath) + promresp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%s", metricsPort) + prometheus.MetricsPrometheusPath) require.NoError(t, err) require.EqualValues(t, http.StatusOK, promresp.StatusCode) @@ -91,6 +87,8 @@ func TestScrapingEndpoint(t *testing.T) { } func TestPanicRecovery(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -100,11 +98,9 @@ func TestPanicRecovery(t *testing.T) { streamPanicInterceptor := func(context.Context, interface{}, *grpc.UnaryServerInfo, grpc.UnaryHandler) (interface{}, error) { panic("test panic") } - port, err := freeport.GetFreePort() - require.NoError(t, err) r := NewSqliteTestRegistry(t, false, WithGRPCUnaryInterceptors(unaryPanicInterceptor), WithGRPCUnaryInterceptors(streamPanicInterceptor)) - require.NoError(t, r.Config(ctx).Set(config.KeyWriteAPIPort, port)) + getAddr := UseDynamicPorts(ctx, t, r) eg := errgroup.Group{} doneShutdown := make(chan struct{}) @@ -112,11 +108,13 @@ func TestPanicRecovery(t *testing.T) { return r.serveWrite(ctx, doneShutdown) }) - conn, err := grpc.DialContext(ctx, fmt.Sprintf("127.0.0.1:%d", port), grpc.WithTransportCredentials(insecure.NewCredentials())) + _, port, _ := getAddr(t, "write") + + conn, err := grpc.DialContext(ctx, fmt.Sprintf("127.0.0.1:%s", port), grpc.WithTransportCredentials(insecure.NewCredentials())) require.NoError(t, err) defer conn.Close() - require.EventuallyWithT(t, func(t *assert.CollectT) { + assert.EventuallyWithT(t, func(t *assert.CollectT) { cl := grpcHealthV1.NewHealthClient(conn) watcher, err := cl.Watch(ctx, &grpcHealthV1.HealthCheckRequest{}) @@ -124,7 +122,7 @@ func TestPanicRecovery(t *testing.T) { require.NoError(t, watcher.CloseSend()) for err := status.Error(codes.Unavailable, "init"); status.Code(err) != codes.Unavailable; _, err = watcher.Recv() { } - }, 2*time.Second, 100*time.Millisecond) + }, 2*time.Second, 10*time.Millisecond) cl := grpcHealthV1.NewHealthClient(conn) // we want to ensure the server is still running after the panic diff --git a/internal/driver/testhelpers.go b/internal/driver/testhelpers.go new file mode 100644 index 000000000..3337c71b1 --- /dev/null +++ b/internal/driver/testhelpers.go @@ -0,0 +1,63 @@ +package driver + +import ( + "context" + "fmt" + "net" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ory/keto/internal/driver/config" +) + +type GetAddr = func(testing.TB, string) (host string, port string, fullAddr string) + +func UseDynamicPorts(ctx context.Context, t testing.TB, r Registry) GetAddr { + t.Helper() + + listenDir := t.TempDir() + readListenFile := fmt.Sprintf("%s/read.addr", listenDir) + writeListenFile := fmt.Sprintf("%s/write.addr", listenDir) + metricsListenFile := fmt.Sprintf("%s/metrics.addr", listenDir) + oplListenFile := fmt.Sprintf("%s/opl.addr", listenDir) + + require.NoError(t, r.Config(ctx).Set(config.KeyReadAPIPort, 0)) + require.NoError(t, r.Config(ctx).Set(config.KeyReadAPIListenFile, "file://"+readListenFile)) + require.NoError(t, r.Config(ctx).Set(config.KeyWriteAPIPort, 0)) + require.NoError(t, r.Config(ctx).Set(config.KeyWriteAPIListenFile, "file://"+writeListenFile)) + require.NoError(t, r.Config(ctx).Set(config.KeyMetricsPort, 0)) + require.NoError(t, r.Config(ctx).Set(config.KeyMetricsListenFile, "file://"+metricsListenFile)) + require.NoError(t, r.Config(ctx).Set(config.KeyOPLSyntaxAPIPort, 0)) + require.NoError(t, r.Config(ctx).Set(config.KeyOPLSyntaxListenFile, "file://"+oplListenFile)) + + return func(t testing.TB, endpoint string) (string, string, string) { + fp := "" + switch endpoint { + case "read": + fp = readListenFile + case "write": + fp = writeListenFile + case "metrics": + fp = metricsListenFile + case "opl": + fp = oplListenFile + default: + t.Fatalf("unknown endpoint: %q", endpoint) + } + + require.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := os.Stat(fp) + require.NoError(t, err) + }, 2*time.Second, 10*time.Millisecond) + + addr, err := os.ReadFile(fp) + require.NoError(t, err) + host, port, err := net.SplitHostPort(string(addr)) + require.NoError(t, err) + return host, port, string(addr) + } +} diff --git a/internal/e2e/full_suit_test.go b/internal/e2e/full_suit_test.go index e00b34387..09a70dd95 100644 --- a/internal/e2e/full_suit_test.go +++ b/internal/e2e/full_suit_test.go @@ -57,24 +57,29 @@ func Test(t *testing.T) { t.Run(fmt.Sprintf("dsn=%s", dsn.Name), func(t *testing.T) { t.Parallel() - ctx, reg, namespaceTestMgr := newInitializedReg(t, dsn, nil) + ctx, reg, namespaceTestMgr, getAddr := newInitializedReg(t, dsn, nil) closeServer := startServer(ctx, t, reg) t.Cleanup(closeServer) + _, _, readAddr := getAddr(t, "read") + _, _, writeAddr := getAddr(t, "write") + _, _, oplAddr := getAddr(t, "opl") + _, _, metricsAddr := getAddr(t, "metrics") + // The test cases start here // We execute every test with all clients available for _, cl := range []client{ &grpcClient{ - readRemote: reg.Config(ctx).ReadAPIListenOn(), - writeRemote: reg.Config(ctx).WriteAPIListenOn(), - oplSyntaxRemote: reg.Config(ctx).OPLSyntaxAPIListenOn(), + readRemote: readAddr, + writeRemote: writeAddr, + oplSyntaxRemote: oplAddr, ctx: ctx, }, &restClient{ - readURL: "http://" + reg.Config(ctx).ReadAPIListenOn(), - writeURL: "http://" + reg.Config(ctx).WriteAPIListenOn(), - oplSyntaxURL: "http://" + reg.Config(ctx).OPLSyntaxAPIListenOn(), + readURL: "http://" + readAddr, + writeURL: "http://" + writeAddr, + oplSyntaxURL: "http://" + oplAddr, }, &cliClient{c: &cmdx.CommandExecuter{ New: func() *cobra.Command { @@ -82,16 +87,16 @@ func Test(t *testing.T) { }, Ctx: ctx, PersistentArgs: []string{ - "--" + cliclient.FlagReadRemote, reg.Config(ctx).ReadAPIListenOn(), - "--" + cliclient.FlagWriteRemote, reg.Config(ctx).WriteAPIListenOn(), + "--" + cliclient.FlagReadRemote, readAddr, + "--" + cliclient.FlagWriteRemote, writeAddr, "--insecure-disable-transport-security=true", "--" + cmdx.FlagFormat, string(cmdx.FormatJSON), }, }}, &sdkClient{ - readRemote: reg.Config(ctx).ReadAPIListenOn(), - writeRemote: reg.Config(ctx).WriteAPIListenOn(), - syntaxRemote: reg.Config(ctx).OPLSyntaxAPIListenOn(), + readRemote: readAddr, + writeRemote: writeAddr, + syntaxRemote: oplAddr, }, } { cl := cl @@ -105,14 +110,14 @@ func Test(t *testing.T) { t.Run("case=metrics are served", func(t *testing.T) { t.Parallel() (&grpcClient{ - readRemote: reg.Config(ctx).ReadAPIListenOn(), - writeRemote: reg.Config(ctx).WriteAPIListenOn(), + readRemote: readAddr, + writeRemote: writeAddr, ctx: ctx, }).waitUntilLive(t) t.Run("case=on "+prometheus.MetricsPrometheusPath, func(t *testing.T) { t.Parallel() - resp, err := http.Get(fmt.Sprintf("http://%s%s", reg.Config(ctx).MetricsListenOn(), prometheus.MetricsPrometheusPath)) + resp, err := http.Get(fmt.Sprintf("http://%s%s", metricsAddr, prometheus.MetricsPrometheusPath)) require.NoError(t, err) require.Equal(t, resp.StatusCode, http.StatusOK) body, err := io.ReadAll(resp.Body) @@ -122,7 +127,7 @@ func Test(t *testing.T) { t.Run("case=not on /", func(t *testing.T) { t.Parallel() - resp, err := http.Get(fmt.Sprintf("http://%s", reg.Config(ctx).MetricsListenOn())) + resp, err := http.Get(fmt.Sprintf("http://%s", metricsAddr)) require.NoError(t, err) require.Equal(t, resp.StatusCode, http.StatusNotFound) }) @@ -134,7 +139,7 @@ func Test(t *testing.T) { func TestServeConfig(t *testing.T) { t.Parallel() - ctx, reg, _ := newInitializedReg(t, dbx.GetSqlite(t, dbx.SQLiteMemory), map[string]interface{}{ + ctx, reg, _, getAddr := newInitializedReg(t, dbx.GetSqlite(t, dbx.SQLiteMemory), map[string]interface{}{ "serve.read.cors.enabled": true, "serve.read.cors.debug": true, "serve.read.cors.allowed_methods": []string{http.MethodGet}, @@ -144,12 +149,14 @@ func TestServeConfig(t *testing.T) { closeServer := startServer(ctx, t, reg) t.Cleanup(closeServer) - for !healthReady(t, "http://"+reg.Config(ctx).ReadAPIListenOn()) { + _, _, readAddr := getAddr(t, "read") + + for !healthReady(t, "http://"+readAddr) { t.Log("Waiting for health check to be ready") time.Sleep(10 * time.Millisecond) } - req, err := http.NewRequest(http.MethodOptions, "http://"+reg.Config(ctx).ReadAPIListenOn()+relationtuple.ReadRouteBase, nil) + req, err := http.NewRequest(http.MethodOptions, "http://"+readAddr+relationtuple.ReadRouteBase, nil) require.NoError(t, err) req.Header.Set("Origin", "https://ory.sh") resp, err := http.DefaultClient.Do(req) diff --git a/internal/e2e/grpc_client_test.go b/internal/e2e/grpc_client_test.go index ea51b7d48..df32c93c3 100644 --- a/internal/e2e/grpc_client_test.go +++ b/internal/e2e/grpc_client_test.go @@ -6,9 +6,10 @@ package e2e import ( "context" "encoding/json" - "google.golang.org/grpc/codes" "time" + "google.golang.org/grpc/codes" + "github.com/ory/keto/ketoapi" opl "github.com/ory/keto/proto/ory/keto/opl/v1alpha1" diff --git a/internal/e2e/helpers.go b/internal/e2e/helpers.go index 96bcef55a..cd175f92f 100644 --- a/internal/e2e/helpers.go +++ b/internal/e2e/helpers.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/ory/x/configx" - "github.com/phayes/freeport" "github.com/spf13/pflag" "github.com/ory/keto/internal/driver/config" @@ -58,15 +57,12 @@ func (m *namespaceTestManager) remove(t *testing.T, name string) { require.NoError(t, m.reg.Config(m.ctx).Set(config.KeyNamespaces, m.nspaces)) } -func newInitializedReg(t testing.TB, dsn *dbx.DsnT, cfgOverwrites map[string]interface{}) (context.Context, driver.Registry, *namespaceTestManager) { +func newInitializedReg(t testing.TB, dsn *dbx.DsnT, cfgOverwrites map[string]interface{}) (context.Context, driver.Registry, *namespaceTestManager, driver.GetAddr) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(func() { cancel() }) - ports, err := freeport.GetFreePorts(4) - require.NoError(t, err) - flags := pflag.NewFlagSet("", pflag.ContinueOnError) configx.RegisterConfigFlag(flags, nil) @@ -75,13 +71,9 @@ func newInitializedReg(t testing.TB, dsn *dbx.DsnT, cfgOverwrites map[string]int "log.level": "debug", "log.leak_sensitive_values": true, config.KeyReadAPIHost: "127.0.0.1", - config.KeyReadAPIPort: ports[0], config.KeyWriteAPIHost: "127.0.0.1", - config.KeyWriteAPIPort: ports[1], config.KeyOPLSyntaxAPIHost: "127.0.0.1", - config.KeyOPLSyntaxAPIPort: ports[2], config.KeyMetricsHost: "127.0.0.1", - config.KeyMetricsPort: ports[3], config.KeyNamespaces: []*namespace.Namespace{}, } for k, v := range cfgOverwrites { @@ -94,6 +86,8 @@ func newInitializedReg(t testing.TB, dsn *dbx.DsnT, cfgOverwrites map[string]int reg, err := driver.NewDefaultRegistry(ctx, flags, true, nil) require.NoError(t, err) + getAddr := driver.UseDynamicPorts(ctx, t, reg) + require.NoError(t, reg.MigrateUp(ctx)) assertMigrated(ctx, t, reg) @@ -101,7 +95,7 @@ func newInitializedReg(t testing.TB, dsn *dbx.DsnT, cfgOverwrites map[string]int reg: reg, ctx: ctx, nspaces: []*namespace.Namespace{}, - } + }, getAddr } func assertMigrated(ctx context.Context, t testing.TB, r driver.Registry) {