diff --git a/.gitignore b/.gitignore index 515701d..91b0982 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ go.work.sum # Release directory release/ -_output/ \ No newline at end of file +_output/ +schema.output.json \ No newline at end of file diff --git a/Makefile b/Makefile index ba0d6dc..c193715 100644 --- a/Makefile +++ b/Makefile @@ -45,4 +45,24 @@ ci-build-cli: clean go run github.com/mitchellh/gox -ldflags '-X github.com/hasura/ndc-rest/ndc-rest-schema/version.BuildVersion=$(VERSION) -s -w -extldflags "-static"' \ -osarch="linux/amd64 darwin/amd64 windows/amd64 darwin/arm64 linux/arm64" \ -output="../$(OUTPUT_DIR)/ndc-rest-schema-{{.OS}}-{{.Arch}}" \ - . \ No newline at end of file + . + +.PHONY: generate-test-config +generate-test-config: + go run ./ndc-rest-schema update -d ./tests/configuration + +.PHONY: start-ddn +start-ddn: + HASURA_DDN_PAT=$$(ddn auth print-pat) docker compose --env-file tests/engine/.env up --build -d + +.PHONY: stop-ddn +stop-ddn: + docker compose down --remove-orphans + +.PHONY: build-supergraph-test +build-supergraph-test: + docker compose up -d --build ndc-rest + cd tests/engine && \ + ddn connector-link update myapi --add-all-resources --subgraph ./app/subgraph.yaml && \ + ddn supergraph build local + make start-ddn \ No newline at end of file diff --git a/README.md b/README.md index 646481b..a8959d0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ files: The config of each element follows the [config schema](https://github.com/hasura/ndc-rest/ndc-rest-schema/blob/main/config.example.yaml) of `ndc-rest-schema`. +You can add many OpenAPI files into the same connector. + > [!IMPORTANT] > Conflicted object and scalar types will be ignored. Only the type of the first file is kept in the schema. @@ -64,29 +66,67 @@ ndc-rest-schema convert -f ./rest/testdata/jsonplaceholder/swagger.json -o ./res - `text/*` - Upload file content types, e.g.`image/*` from `base64` arguments. -### Environment variable template +### Authentication -The connector can replaces `{{xxx}}` templates with environment variables. The converter automatically renders variables for API keys and tokens when converting OpenAPI documents. However, you can add your custom variables as well. +The current version supports API key and Auth token authentication schemes. The configuration is inspired from `securitySchemes` [with env variables](https://github.com/hasura/ndc-rest/ndc-rest-schema#authentication). The connector supports the following authentication strategies: -### Timeout and retry +- API Key +- Bearer Auth +- Cookies +- OAuth 2.0 + +The configuration automatically generates environment variables for the api key and Bearer token. + +For Cookie authentication and OAuth 2.0, you need to enable headers forwarding from the Hasura engine to the connector. -The global timeout and retry strategy can be configured in the `settings` object. +### Header Forwarding + +Enable `forwardHeaders` in the configuration file. ```yaml -settings: - timeout: 30 - retry: - times: 2 - # delay between each retry in milliseconds - delay: 1000 - httpStatus: [429, 500, 502, 503] +# ... +forwardHeaders: + enabled: true + argumentField: headers ``` -### Authentication +And configure in the connector link metadata. + +```yaml +kind: DataConnectorLink +version: v1 +definition: + name: my_api + # ... + argumentPresets: + - argument: headers + value: + httpHeaders: + forward: + - Cookie + additional: {} +``` + +See the configuration example in [Hasura docs](https://hasura.io/docs/3.0/recipes/business-logic/http-header-forwarding/#step-2-update-the-metadata-1). + +### Timeout and retry -The current version supports API key and Auth token authentication schemes. The configuration is inspired from `securitySchemes` [with env variables](https://github.com/hasura/ndc-rest/ndc-rest-schema#authentication) +The global timeout and retry strategy can be configured in each file: -See [this example](rest/testdata/auth/schema.yaml) for more context. +```yaml +files: + - file: swagger.json + spec: oas2 + timeout: + value: 30 + retry: + times: + value: 1 + delay: + # delay between each retry in milliseconds + value: 500 + httpStatus: [429, 500, 502, 503] +``` ## Distributed execution diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..6096e70 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,15 @@ +include: + - tests/engine/compose.yaml +services: + ndc-rest: + build: + context: . + ports: + - 8080:8080 + volumes: + - ./tests/configuration:/etc/connector:ro + extra_hosts: + - local.hasura.dev=host-gateway + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://local.hasura.dev:4317 + HASURA_LOG_LEVEL: debug diff --git a/connector-definition/config.yaml b/connector-definition/config.yaml index ef0f631..98d7c0b 100644 --- a/connector-definition/config.yaml +++ b/connector-definition/config.yaml @@ -11,7 +11,7 @@ concurrency: rest: 5 files: - file: https://raw.githubusercontent.com/hasura/ndc-rest/main/connector/testdata/jsonplaceholder/swagger.json - spec: openapi2 + spec: oas2 timeout: value: 30 retry: diff --git a/connector/connector.go b/connector/connector.go index 42fb384..d19c46c 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -42,8 +42,11 @@ func (c *RESTConnector) ParseConfiguration(ctx context.Context, configurationDir Query: schema.QueryCapabilities{ Variables: schema.LeafCapability{}, NestedFields: schema.NestedFieldCapabilities{}, + Explain: schema.LeafCapability{}, + }, + Mutation: schema.MutationCapabilities{ + Explain: schema.LeafCapability{}, }, - Mutation: schema.MutationCapabilities{}, }, } rawCapabilities, err := json.Marshal(restCapabilities) diff --git a/connector/internal/client.go b/connector/internal/client.go index 8509836..27ec1a6 100644 --- a/connector/internal/client.go +++ b/connector/internal/client.go @@ -4,11 +4,14 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "log/slog" "math" "net/http" + "slices" + "strconv" "strings" "sync" "time" @@ -58,7 +61,7 @@ func (client *HTTPClient) Send(ctx context.Context, request *RetryableRequest, s } if !restOptions.Distributed { - result, headers, err := client.sendSingle(ctx, &requests[0], selection, resultType) + result, headers, err := client.sendSingle(ctx, &requests[0], selection, resultType, requests[0].ServerID, "single") if err != nil { return nil, nil, err } @@ -83,7 +86,7 @@ func (client *HTTPClient) sendSequence(ctx context.Context, requests []Retryable var firstHeaders http.Header for _, req := range requests { - result, headers, err := client.sendSingle(ctx, &req, selection, resultType) + result, headers, err := client.sendSingle(ctx, &req, selection, resultType, req.ServerID, "sequence") if err != nil { results.Errors = append(results.Errors, DistributedError{ Server: req.ServerID, @@ -116,7 +119,7 @@ func (client *HTTPClient) sendParallel(ctx context.Context, requests []Retryable sendFunc := func(req RetryableRequest) { eg.Go(func() error { - result, headers, err := client.sendSingle(ctx, &req, selection, resultType) + result, headers, err := client.sendSingle(ctx, &req, selection, resultType, req.ServerID, "parallel") lock.Lock() defer lock.Unlock() if err != nil { @@ -148,25 +151,33 @@ func (client *HTTPClient) sendParallel(ctx context.Context, requests []Retryable } // execute a request to the remote server with retries -func (client *HTTPClient) sendSingle(ctx context.Context, request *RetryableRequest, selection schema.NestedField, resultType schema.Type) (any, http.Header, *schema.ConnectorError) { - ctx, span := client.tracer.Start(ctx, "request_remote_server") +func (client *HTTPClient) sendSingle(ctx context.Context, request *RetryableRequest, selection schema.NestedField, resultType schema.Type, serverID string, mode string) (any, http.Header, *schema.ConnectorError) { + ctx, span := client.tracer.Start(ctx, fmt.Sprintf("Send Request to Server %s", serverID)) defer span.End() - span.SetAttributes( - attribute.String("request_url", request.URL), - attribute.String("method", request.RawRequest.Method), - ) + + span.SetAttributes(attribute.String("execution.mode", mode)) + + requestURL := request.URL.String() + rawPort := request.URL.Port() + port := 80 + if rawPort != "" { + if p, err := strconv.ParseInt(rawPort, 10, 32); err == nil { + port = int(p) + } + } else if strings.HasPrefix(request.URL.Scheme, "https") { + port = 443 + } + client.propagator.Inject(ctx, propagation.HeaderCarrier(request.Headers)) - var resp *http.Response - var err error logger := connector.GetLogger(ctx) - if logger.Enabled(ctx, slog.LevelDebug) { logAttrs := []any{ - slog.String("request_url", request.URL), + slog.String("request_url", requestURL), slog.String("request_method", request.RawRequest.Method), slog.Any("request_headers", request.Headers), } + if request.Body != nil { bs, _ := io.ReadAll(request.Body) logAttrs = append(logAttrs, slog.String("request_body", string(bs))) @@ -174,97 +185,129 @@ func (client *HTTPClient) sendSingle(ctx context.Context, request *RetryableRequ logger.Debug("sending request to remote server...", logAttrs...) } + var resp *http.Response + var errorBytes []byte + var err error + var cancel context.CancelFunc + times := int(math.Max(float64(request.Runtime.Retry.Times), 1)) delayMs := int(math.Max(float64(request.Runtime.Retry.Delay), 100)) for i := range times { - req, cancel, reqError := request.CreateRequest(ctx) - if reqError != nil { - cancel() - span.SetStatus(codes.Error, "error happened when creating request") + resp, errorBytes, cancel, err = client.doRequest(ctx, request, port, i) + if err != nil { + span.SetStatus(codes.Error, "failed to execute the request") span.RecordError(err) + return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, err.Error(), nil) } - resp, err = client.client.Do(req) - if (err == nil && resp.StatusCode >= 200 && resp.StatusCode < 299) || i >= times { - break - } - logAttrs := []any{} - if err != nil { - span.AddEvent(fmt.Sprintf("request error, retry %d of %d", i+1, times), trace.WithAttributes(attribute.String("error", err.Error()))) - logAttrs = append(logAttrs, slog.Any("error", err.Error())) - } else { - var respBody []byte - if resp.Body != nil { - respBody, _ = io.ReadAll(resp.Body) - _ = resp.Body.Close() - } - - logAttrs = append(logAttrs, - slog.Int("http_status", resp.StatusCode), - slog.Any("response_headers", resp.Header), - slog.String("response_body", string(respBody)), - ) - span.AddEvent( - fmt.Sprintf("received error from remote server, retry %d of %d", i+1, times), - trace.WithAttributes( - attribute.Int("http_status", resp.StatusCode), - attribute.String("response_body", string(respBody)), - ), - ) + if (resp.StatusCode >= 200 && resp.StatusCode < 299) || + !slices.Contains(request.Runtime.Retry.HTTPStatus, resp.StatusCode) || i >= times { + break } if logger.Enabled(ctx, slog.LevelDebug) { logger.Debug( fmt.Sprintf("received error from remote server, retry %d of %d...", i+1, times), - logAttrs..., + slog.Int("http_status", resp.StatusCode), + slog.Any("response_headers", resp.Header), + slog.String("response_body", string(errorBytes)), ) } time.Sleep(time.Duration(delayMs) * time.Millisecond) } - if err != nil { - span.SetStatus(codes.Error, "error happened when creating request") - span.RecordError(err) - return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, err.Error(), nil) - } - - span.SetAttributes(attribute.Int("http_status", resp.StatusCode)) - - return evalHTTPResponse(ctx, span, resp, selection, resultType) -} + defer cancel() -func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, selection schema.NestedField, resultType schema.Type) (any, http.Header, *schema.ConnectorError) { - logger := connector.GetLogger(ctx) contentType := parseContentType(resp.Header.Get(contentTypeHeader)) if resp.StatusCode >= 400 { - var respBody []byte - if resp.Body != nil { - var err error - respBody, err = io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if err != nil { - span.SetStatus(codes.Error, "error happened when reading response body") - span.RecordError(err) - return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, resp.Status, map[string]any{ - "error": err, - }) - } - } details := make(map[string]any) - if contentType == rest.ContentTypeJSON && json.Valid(respBody) { - details["error"] = json.RawMessage(respBody) + if contentType == rest.ContentTypeJSON && json.Valid(errorBytes) { + details["error"] = json.RawMessage(errorBytes) } else { - details["error"] = string(respBody) + details["error"] = string(errorBytes) } - span.SetAttributes(attribute.String("response_error", string(respBody))) span.SetStatus(codes.Error, "received error from remote server") + return nil, nil, schema.NewConnectorError(resp.StatusCode, resp.Status, details) } + result, headers, evalErr := evalHTTPResponse(ctx, span, resp, contentType, selection, resultType, logger) + if evalErr != nil { + span.SetStatus(codes.Error, "failed to decode the http response") + span.RecordError(evalErr) + + return nil, nil, evalErr + } + + return result, headers, nil +} + +func (client *HTTPClient) doRequest(ctx context.Context, request *RetryableRequest, port int, retryCount int) (*http.Response, []byte, context.CancelFunc, error) { + method := strings.ToUpper(request.RawRequest.Method) + ctx, span := client.tracer.Start(ctx, fmt.Sprintf("%s %s", method, request.RawRequest.URL), trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() + + requestURL := request.URL.String() + + span.SetAttributes( + attribute.String("db.system", "rest"), + attribute.String("http.request.method", method), + attribute.String("url.full", requestURL), + attribute.String("server.address", request.URL.Hostname()), + attribute.Int("server.port", port), + attribute.String("network.protocol.name", "http"), + ) + + if request.ContentLength > 0 { + span.SetAttributes(attribute.Int64("http.request.body.size", request.ContentLength)) + } + if retryCount > 0 { + span.SetAttributes(attribute.Int("http.request.resend_count", retryCount)) + } + setHeaderAttributes(span, "http.request.header.", request.Headers) + + client.propagator.Inject(ctx, propagation.HeaderCarrier(request.Headers)) + + req, cancel, err := request.CreateRequest(ctx) + if err != nil { + span.SetStatus(codes.Error, "error happened when creating request") + span.RecordError(err) + + return nil, nil, nil, err + } + + resp, err := client.client.Do(req) + if err != nil { + span.SetStatus(codes.Error, "error happened when executing the request") + span.RecordError(err) + cancel() + + return nil, nil, nil, err + } + + span.SetAttributes(attribute.Int("http.response.status_code", resp.StatusCode)) + setHeaderAttributes(span, "http.response.header.", resp.Header) + + if resp.StatusCode < 300 { + return resp, nil, cancel, nil + } + + defer resp.Body.Close() + span.SetStatus(codes.Error, "Non-2xx status") + body, err := io.ReadAll(resp.Body) + if err != nil { + span.RecordError(err) + } else { + span.RecordError(errors.New(string(body))) + } + + return resp, body, cancel, nil +} + +func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, contentType string, selection schema.NestedField, resultType schema.Type, logger *slog.Logger) (any, http.Header, *schema.ConnectorError) { if logger.Enabled(ctx, slog.LevelDebug) { logAttrs := []any{ slog.Int("http_status", resp.StatusCode), @@ -273,9 +316,11 @@ func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, if resp.Body != nil && resp.StatusCode != http.StatusNoContent { respBody, readErr := io.ReadAll(resp.Body) _ = resp.Body.Close() + if readErr != nil { span.SetStatus(codes.Error, "error happened when reading response body") span.RecordError(readErr) + return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, "error happened when reading response body", map[string]any{ "error": readErr.Error(), }) @@ -283,9 +328,16 @@ func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, resp.Body = io.NopCloser(bytes.NewBuffer(respBody)) logAttrs = append(logAttrs, slog.String("response_body", string(respBody))) } + logger.Debug("received response from remote server", logAttrs...) } + defer func() { + if resp.Body != nil { + _ = resp.Body.Close() + } + }() + if resp.StatusCode == http.StatusNoContent { return true, resp.Header, nil } @@ -294,10 +346,6 @@ func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, return nil, resp.Header, nil } - defer func() { - _ = resp.Body.Close() - }() - switch contentType { case "": respBody, err := io.ReadAll(resp.Body) @@ -319,7 +367,6 @@ func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, namedType, err := resultType.AsNamed() if err == nil && namedType.Name == string(rest.ScalarString) { respBytes, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() if err != nil { return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, "failed to read response", map[string]any{ "reason": err.Error(), @@ -337,7 +384,6 @@ func evalHTTPResponse(ctx context.Context, span trace.Span, resp *http.Response, var result any err := json.NewDecoder(resp.Body).Decode(&result) - _ = resp.Body.Close() if err != nil { return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, err.Error(), nil) } diff --git a/connector/internal/request.go b/connector/internal/request.go index 5a37958..23f9381 100644 --- a/connector/internal/request.go +++ b/connector/internal/request.go @@ -18,13 +18,14 @@ import ( // RetryableRequest wraps the raw request with retryable type RetryableRequest struct { - RawRequest *rest.Request - URL string - ServerID string - ContentType string - Headers http.Header - Body io.ReadSeeker - Runtime rest.RuntimeSettings + RawRequest *rest.Request + URL url.URL + ServerID string + ContentType string + ContentLength int64 + Headers http.Header + Body io.ReadSeeker + Runtime rest.RuntimeSettings } // CreateRequest creates an HTTP request with body copied @@ -40,10 +41,12 @@ func (r *RetryableRequest) CreateRequest(ctx context.Context) (*http.Request, co if timeout == 0 { timeout = defaultTimeoutSeconds } + ctxR, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) - request, err := http.NewRequestWithContext(ctxR, strings.ToUpper(r.RawRequest.Method), r.URL, r.Body) + request, err := http.NewRequestWithContext(ctxR, strings.ToUpper(r.RawRequest.Method), r.URL.String(), r.Body) if err != nil { cancel() + return nil, nil, err } for key, header := range r.Headers { @@ -54,15 +57,15 @@ func (r *RetryableRequest) CreateRequest(ctx context.Context) (*http.Request, co return request, cancel, nil } -func getHostFromServers(servers []rest.ServerConfig, serverIDs []string) (string, string) { - var results []string +func getBaseURLFromServers(servers []rest.ServerConfig, serverIDs []string) (*url.URL, string) { + var results []url.URL var selectedServerIDs []string for _, server := range servers { if len(serverIDs) > 0 && !slices.Contains(serverIDs, server.ID) { continue } - hostPtr := server.GetURL() - if hostPtr != "" { + hostPtr, err := server.GetURL() + if err == nil { results = append(results, hostPtr) selectedServerIDs = append(selectedServerIDs, server.ID) } @@ -70,24 +73,28 @@ func getHostFromServers(servers []rest.ServerConfig, serverIDs []string) (string switch len(results) { case 0: - return "", "" + return nil, "" case 1: - return results[0], selectedServerIDs[0] + result := results[0] + return &result, selectedServerIDs[0] default: index := rand.IntN(len(results) - 1) - return results[index], selectedServerIDs[index] + host := results[index] + return &host, selectedServerIDs[index] } } // BuildDistributedRequestsWithOptions builds distributed requests with options func BuildDistributedRequestsWithOptions(request *RetryableRequest, restOptions *RESTOptions) ([]RetryableRequest, error) { - if strings.HasPrefix(request.URL, "http") { + if strings.HasPrefix(request.URL.Scheme, "http") { return []RetryableRequest{*request}, nil } if !restOptions.Distributed || len(restOptions.Settings.Servers) == 1 { - host, serverID := getHostFromServers(restOptions.Settings.Servers, restOptions.Servers) - request.URL = host + request.URL + baseURL, serverID := getBaseURLFromServers(restOptions.Settings.Servers, restOptions.Servers) + request.URL.Scheme = baseURL.Scheme + request.URL.Host = baseURL.Host + request.URL.Path = baseURL.Path + request.URL.Path request.ServerID = serverID if err := request.applySettings(restOptions.Settings, restOptions.Explain); err != nil { return nil, err @@ -112,13 +119,15 @@ func BuildDistributedRequestsWithOptions(request *RetryableRequest, restOptions } } for _, serverID := range serverIDs { - host, serverID := getHostFromServers(restOptions.Settings.Servers, []string{serverID}) - if host == "" { + baseURL, serverID := getBaseURLFromServers(restOptions.Settings.Servers, []string{serverID}) + if baseURL == nil { continue } - + baseURL.Path += request.URL.Path + baseURL.RawQuery = request.URL.RawQuery + baseURL.Fragment = request.URL.Fragment req := RetryableRequest{ - URL: host + request.URL, + URL: *baseURL, ServerID: serverID, RawRequest: request.RawRequest, ContentType: request.ContentType, @@ -215,15 +224,11 @@ func (req *RetryableRequest) applySecurity(serverConfig *rest.ServerConfig, isEx case rest.APIKeyInQuery: value := securityScheme.GetValue() if value != "" { - endpoint, err := url.Parse(req.URL) - if err != nil { - return err - } - + endpoint := req.URL q := endpoint.Query() q.Add(securityScheme.Name, eitherMaskSecret(value, isExplain)) endpoint.RawQuery = q.Encode() - req.URL = endpoint.String() + req.URL = endpoint } case rest.APIKeyInCookie: // Cookie header should be forwarded from Hasura engine diff --git a/connector/internal/request_builder.go b/connector/internal/request_builder.go index 9753580..5b7648e 100644 --- a/connector/internal/request_builder.go +++ b/connector/internal/request_builder.go @@ -43,11 +43,16 @@ func (c *RequestBuilder) Build() (*RetryableRequest, error) { }) } - var buffer io.ReadSeeker - rawRequest := c.Operation.Request contentType := rest.ContentTypeJSON + request := &RetryableRequest{ + URL: *endpoint, + RawRequest: rawRequest, + Headers: headers, + Runtime: c.Runtime, + } + if rawRequest.RequestBody != nil { contentType = rawRequest.RequestBody.ContentType bodyInfo, infoOk := c.Operation.Arguments[rest.BodyKey] @@ -64,28 +69,37 @@ func (c *RequestBuilder) Build() (*RetryableRequest, error) { if err != nil { return nil, err } - buffer = bytes.NewReader([]byte(dataURI.Data)) + r := bytes.NewReader([]byte(dataURI.Data)) + request.ContentLength = r.Size() + request.Body = r } else if strings.HasPrefix(contentType, "text/") { - buffer = bytes.NewReader([]byte(fmt.Sprint(bodyData))) + r := bytes.NewReader([]byte(fmt.Sprint(bodyData))) + request.ContentLength = r.Size() + request.Body = r } else if strings.HasPrefix(contentType, "multipart/") { - buffer, contentType, err = c.createMultipartForm(bodyData) + var r *bytes.Reader + r, contentType, err = c.createMultipartForm(bodyData) if err != nil { return nil, err } + request.ContentLength = r.Size() + request.Body = r } else { switch contentType { case rest.ContentTypeFormURLEncoded: - buffer, err = c.createFormURLEncoded(&bodyInfo, bodyData) + r, err := c.createFormURLEncoded(&bodyInfo, bodyData) if err != nil { return nil, err } + request.Body = r case rest.ContentTypeJSON, "": bodyBytes, err := json.Marshal(bodyData) if err != nil { return nil, err } - buffer = bytes.NewReader(bodyBytes) + request.ContentLength = int64(len(bodyBytes)) + request.Body = bytes.NewReader(bodyBytes) default: return nil, fmt.Errorf("unsupported content type %s", contentType) } @@ -101,14 +115,7 @@ func (c *RequestBuilder) Build() (*RetryableRequest, error) { } } - request := &RetryableRequest{ - URL: endpoint, - RawRequest: rawRequest, - ContentType: contentType, - Headers: headers, - Body: buffer, - Runtime: c.Runtime, - } + request.ContentType = contentType if rawRequest.RuntimeSettings != nil { if rawRequest.RuntimeSettings.Timeout > 0 { @@ -155,7 +162,7 @@ func (c *RequestBuilder) createFormURLEncoded(bodyInfo *rest.ArgumentInfo, bodyD return bytes.NewReader([]byte(rawQuery)), nil } -func (c *RequestBuilder) createMultipartForm(bodyData any) (io.ReadSeeker, string, error) { +func (c *RequestBuilder) createMultipartForm(bodyData any) (*bytes.Reader, string, error) { bodyInfo, ok := c.Operation.Arguments[rest.BodyKey] if !ok { return nil, "", errRequestBodyTypeRequired diff --git a/connector/internal/request_parameter.go b/connector/internal/request_parameter.go index 4e22a20..34fd25f 100644 --- a/connector/internal/request_parameter.go +++ b/connector/internal/request_parameter.go @@ -21,16 +21,16 @@ import ( var urlAndHeaderLocations = []rest.ParameterLocation{rest.InPath, rest.InQuery, rest.InHeader} // evaluate URL and header parameters -func (c *RequestBuilder) evalURLAndHeaderParameters() (string, http.Header, error) { +func (c *RequestBuilder) evalURLAndHeaderParameters() (*url.URL, http.Header, error) { endpoint, err := url.Parse(c.Operation.Request.URL) if err != nil { - return "", nil, err + return nil, nil, err } headers := http.Header{} for k, h := range c.Operation.Request.Headers { v, err := h.Get() if err != nil { - return "", nil, fmt.Errorf("invalid header value, key: %s, %w", k, err) + return nil, nil, fmt.Errorf("invalid header value, key: %s, %w", k, err) } if v != "" { headers.Add(k, v) @@ -42,10 +42,10 @@ func (c *RequestBuilder) evalURLAndHeaderParameters() (string, http.Header, erro continue } if err := c.evalURLAndHeaderParameterBySchema(endpoint, &headers, argumentKey, &argumentInfo, c.Arguments[argumentKey]); err != nil { - return "", nil, fmt.Errorf("%s: %w", argumentKey, err) + return nil, nil, fmt.Errorf("%s: %w", argumentKey, err) } } - return endpoint.String(), headers, nil + return endpoint, headers, nil } // the query parameters serialization follows [OAS 3.1 spec] diff --git a/connector/internal/request_parameter_test.go b/connector/internal/request_parameter_test.go index 0e5c070..23dba43 100644 --- a/connector/internal/request_parameter_test.go +++ b/connector/internal/request_parameter_test.go @@ -301,7 +301,7 @@ func TestEvalURLAndHeaderParameters(t *testing.T) { assert.ErrorContains(t, err, tc.errorMsg) } else { assert.NilError(t, err) - decodedValue, err := url.QueryUnescape(result) + decodedValue, err := url.QueryUnescape(result.String()) assert.NilError(t, err) assert.Equal(t, tc.expectedURL, decodedValue) for k, v := range tc.headers { diff --git a/connector/internal/types.go b/connector/internal/types.go index e403488..0c1a7a5 100644 --- a/connector/internal/types.go +++ b/connector/internal/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" rest "github.com/hasura/ndc-rest/ndc-rest-schema/schema" "github.com/hasura/ndc-sdk-go/schema" @@ -23,6 +24,7 @@ var ( ) var defaultRetryHTTPStatus = []int{429, 500, 502, 503} +var sensitiveHeaderRegex = regexp.MustCompile(`auth|key|secret|token`) // RESTOptions represent execution options for REST requests type RESTOptions struct { diff --git a/connector/internal/utils.go b/connector/internal/utils.go index 93858d1..2a33f47 100644 --- a/connector/internal/utils.go +++ b/connector/internal/utils.go @@ -2,9 +2,12 @@ package internal import ( "fmt" + "net/http" "strings" "github.com/hasura/ndc-sdk-go/schema" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // UnwrapNullableType unwraps the underlying type of the nullable type @@ -43,3 +46,19 @@ func MaskString(input string) string { return input[0:3] + strings.Repeat("*", 7) + fmt.Sprintf("(%d)", inputLength) } } + +func setHeaderAttributes(span trace.Span, prefix string, httpHeaders http.Header) { + for key, headers := range httpHeaders { + if len(headers) == 0 { + continue + } + values := headers + if sensitiveHeaderRegex.MatchString(strings.ToLower(key)) { + values = make([]string, len(headers)) + for i, header := range headers { + values[i] = MaskString(header) + } + } + span.SetAttributes(attribute.StringSlice(prefix+strings.ToLower(key), values)) + } +} diff --git a/connector/query.go b/connector/query.go index 7f2fdaa..384e9a1 100644 --- a/connector/query.go +++ b/connector/query.go @@ -149,6 +149,7 @@ func (c *RESTConnector) execQuery(ctx context.Context, state *State, request *sc if err != nil { span.SetStatus(codes.Error, "failed to explain query") span.RecordError(err) + return nil, err } @@ -185,7 +186,7 @@ func serializeExplainResponse(httpRequest *internal.RetryableRequest, restOption if err != nil { return nil, err } - explainResp.Details["url"] = requests[0].URL + explainResp.Details["url"] = requests[0].URL.String() if httpRequest.Body != nil { bodyBytes, err := io.ReadAll(httpRequest.Body) diff --git a/ndc-rest-schema/README.md b/ndc-rest-schema/README.md index 0f41f3e..55ec110 100644 --- a/ndc-rest-schema/README.md +++ b/ndc-rest-schema/README.md @@ -92,7 +92,8 @@ The URL can be a relative path or an absolute URL. If the URL is relative, there ```yaml settings: servers: - - url: http://petstore.swagger.io/v1 + - url: + value: http://petstore.swagger.io/v1 ``` `parameters` include the list of URL and query parameters so the connector can replace values from request arguments. @@ -121,14 +122,17 @@ Environment variable template which is in `{{CONSTANT_CASE}}` or `{{CONSTANT_CAS ```yaml settings: servers: - - url: "{{PET_STORE_SERVER_URL:-https://petstore3.swagger.io/api/v3}}" + - url: + env: PET_STORE_SERVER_URL + value: https://petstore3.swagger.io/api/v3 timeout: 30 headers: foo: bar securitySchemes: api_key: type: apiKey - value: "{{PET_STORE_API_KEY}}" + value: + env: PET_STORE_API_KEY in: header name: api_key petstore_auth: @@ -257,7 +261,9 @@ There is an extra `value` field with the environment variable template to be rep "securitySchemes": { "api_key": { "type": "apiKey", - "value": "{{API_KEY}}", // the constant case of api_key + "value": { + "env": "API_KEY" // the constant case of api_key + }, "in": "header", "name": "api_key" } @@ -277,7 +283,9 @@ This is the general authentication for [Basic](https://swagger.io/docs/specifica "bearer_auth": { "type": "http", "scheme": "bearer", - "value": "{{BEARER_AUTH_TOKEN}}", // the constant case of bearer_auth + _TOKEN suffix + "value": { + "env": "BEARER_AUTH_TOKEN" // the constant case of bearer_auth + _TOKEN suffix + }, "header": "Authentication" } } diff --git a/ndc-rest-schema/command/testdata/.gitignore b/ndc-rest-schema/command/testdata/.gitignore deleted file mode 100644 index 5a518a5..0000000 --- a/ndc-rest-schema/command/testdata/.gitignore +++ /dev/null @@ -1 +0,0 @@ -schema.output.json \ No newline at end of file diff --git a/ndc-rest-schema/command/testdata/auth/config.yaml b/ndc-rest-schema/command/testdata/auth/config.yaml index cb04189..1f2eabc 100644 --- a/ndc-rest-schema/command/testdata/auth/config.yaml +++ b/ndc-rest-schema/command/testdata/auth/config.yaml @@ -1,10 +1,14 @@ # yaml-language-server: $schema=../../../jsonschema/configuration.schema.json +output: schema.output.json strict: true forwardHeaders: enabled: false argumentField: headers - responseHeadersField: null - responseResultField: null + responseHeaders: null +concurrency: + query: 1 + mutation: 1 + rest: 10 files: - file: schema.yaml spec: ndc diff --git a/ndc-rest-schema/command/testdata/auth/expected.json b/ndc-rest-schema/command/testdata/auth/expected.json new file mode 100644 index 0000000..d08dd27 --- /dev/null +++ b/ndc-rest-schema/command/testdata/auth/expected.json @@ -0,0 +1,318 @@ +[ + { + "name": "testdata/auth/schema.yaml", + "settings": { + "servers": [ + { + "url": { + "env": "PET_STORE_URL" + }, + "securitySchemes": { + "api_key": { + "type": "apiKey", + "value": { + "env": "PET_STORE_API_KEY" + }, + "in": "header", + "name": "api_key" + }, + "bearer": { + "type": "http", + "value": { + "env": "PET_STORE_BEARER_TOKEN" + }, + "header": "", + "scheme": "bearer" + }, + "petstore_auth": { + "type": "oauth2", + "flows": { + "implicit": { + "authorizationUrl": "https://petstore3.swagger.io/oauth/authorize", + "scopes": { + "read:pets": "read your pets", + "write:pets": "modify pets in your account" + } + } + } + } + }, + "security": [ + { + "api_key": [] + } + ] + } + ], + "securitySchemes": { + "api_key": { + "type": "apiKey", + "value": { + "env": "PET_STORE_API_KEY" + }, + "in": "header", + "name": "api_key" + }, + "bearer": { + "type": "http", + "value": { + "env": "PET_STORE_BEARER_TOKEN" + }, + "header": "", + "scheme": "bearer" + }, + "petstore_auth": { + "type": "oauth2", + "flows": { + "implicit": { + "authorizationUrl": "https://petstore3.swagger.io/oauth/authorize", + "scopes": { + "read:pets": "read your pets", + "write:pets": "modify pets in your account" + } + } + } + } + }, + "security": [ + { + "api_key": [] + } + ], + "version": "1.0.18" + }, + "functions": { + "findPets": { + "request": { + "url": "/pet", + "method": "get", + "type": "rest", + "response": { + "contentType": "" + } + }, + "arguments": {}, + "description": "Finds Pets", + "result_type": { + "element_type": { + "name": "Pet", + "type": "named" + }, + "type": "array" + } + }, + "findPetsByStatus": { + "request": { + "url": "/pet/findByStatus", + "method": "get", + "type": "rest", + "security": [ + { + "bearer": [] + } + ], + "response": { + "contentType": "" + } + }, + "arguments": { + "status": { + "description": "Status values that need to be considered for filter", + "type": { + "type": "nullable", + "underlying_type": { + "name": "String", + "type": "named" + } + }, + "rest": { + "in": "query", + "schema": { + "type": ["string"] + } + } + } + }, + "description": "Finds Pets by status", + "result_type": { + "element_type": { + "name": "Pet", + "type": "named" + }, + "type": "array" + } + }, + "petRetry": { + "request": { + "url": "/pet/retry", + "method": "get", + "type": "rest", + "response": { + "contentType": "" + } + }, + "arguments": {}, + "result_type": { + "element_type": { + "name": "Pet", + "type": "named" + }, + "type": "array" + } + } + }, + "object_types": { + "CreateModelRequest": { + "fields": { + "model": { + "description": "The name of the model to create", + "type": { + "type": "nullable", + "underlying_type": { + "name": "String", + "type": "named" + } + } + } + } + }, + "Pet": { + "fields": { + "id": { + "type": { + "type": "nullable", + "underlying_type": { + "name": "Int", + "type": "named" + } + } + }, + "name": { + "type": { + "name": "String", + "type": "named" + } + } + } + }, + "ProgressResponse": { + "fields": { + "completed": { + "description": "The completed size of the task", + "type": { + "type": "nullable", + "underlying_type": { + "name": "Int32", + "type": "named" + } + } + }, + "status": { + "description": "The status of the request", + "type": { + "type": "nullable", + "underlying_type": { + "name": "String", + "type": "named" + } + } + } + } + } + }, + "procedures": { + "addPet": { + "request": { + "url": "/pet", + "method": "post", + "type": "rest", + "headers": { + "Content-Type": { + "value": "application/json" + } + }, + "security": [ + { + "api_key": [] + } + ], + "requestBody": { + "contentType": "application/json" + }, + "response": { + "contentType": "" + } + }, + "arguments": { + "body": { + "description": "Request body of /pet", + "type": { + "name": "Pet", + "type": "named" + }, + "rest": { + "in": "body" + } + } + }, + "description": "Add a new pet to the store", + "result_type": { + "name": "Pet", + "type": "named" + } + }, + "createModel": { + "request": { + "url": "/model", + "method": "post", + "type": "rest", + "requestBody": { + "contentType": "application/json" + }, + "response": { + "contentType": "application/x-ndjson" + } + }, + "arguments": { + "body": { + "description": "Request body of POST /api/create", + "type": { + "name": "CreateModelRequest", + "type": "named" + } + } + }, + "result_type": { + "element_type": { + "name": "ProgressResponse", + "type": "named" + }, + "type": "array" + } + } + }, + "scalar_types": { + "Boolean": { + "aggregate_functions": {}, + "comparison_operators": {}, + "representation": { + "type": "boolean" + } + }, + "Int": { + "aggregate_functions": {}, + "comparison_operators": {}, + "representation": { + "type": "int32" + } + }, + "String": { + "aggregate_functions": {}, + "comparison_operators": {}, + "representation": { + "type": "string" + } + } + } + } +] diff --git a/ndc-rest-schema/command/testdata/auth/schema.yaml b/ndc-rest-schema/command/testdata/auth/schema.yaml index f8c6ba5..d9fe12c 100644 --- a/ndc-rest-schema/command/testdata/auth/schema.yaml +++ b/ndc-rest-schema/command/testdata/auth/schema.yaml @@ -171,12 +171,15 @@ scalar_types: Boolean: aggregate_functions: {} comparison_operators: {} + representation: + type: boolean Int: aggregate_functions: {} comparison_operators: {} - JSON: - aggregate_functions: {} - comparison_operators: {} + representation: + type: int32 String: aggregate_functions: {} comparison_operators: {} + representation: + type: string diff --git a/ndc-rest-schema/command/testdata/patch/expected.json b/ndc-rest-schema/command/testdata/patch/expected.json index bcd2c1d..cdfd62a 100644 --- a/ndc-rest-schema/command/testdata/patch/expected.json +++ b/ndc-rest-schema/command/testdata/patch/expected.json @@ -1139,15 +1139,24 @@ "scalar_types": { "Boolean": { "aggregate_functions": {}, - "comparison_operators": {} + "comparison_operators": {}, + "representation": { + "type": "boolean" + } }, "Int": { "aggregate_functions": {}, - "comparison_operators": {} + "comparison_operators": {}, + "representation": { + "type": "int32" + } }, "JSON": { "aggregate_functions": {}, - "comparison_operators": {} + "comparison_operators": {}, + "representation": { + "type": "json" + } }, "RestServerId": { "aggregate_functions": {}, @@ -1159,7 +1168,10 @@ }, "String": { "aggregate_functions": {}, - "comparison_operators": {} + "comparison_operators": {}, + "representation": { + "type": "string" + } } } } diff --git a/ndc-rest-schema/command/update_test.go b/ndc-rest-schema/command/update_test.go index ba40515..66141d4 100644 --- a/ndc-rest-schema/command/update_test.go +++ b/ndc-rest-schema/command/update_test.go @@ -14,8 +14,7 @@ import ( "gotest.tools/v3/assert" ) -func TestOpenAPIv3ToRESTSchema(t *testing.T) { - +func TestUpdateCommand(t *testing.T) { testCases := []struct { Argument UpdateCommandArguments Expected string @@ -27,6 +26,13 @@ func TestOpenAPIv3ToRESTSchema(t *testing.T) { }, Expected: "testdata/patch/expected.json", }, + // go run ./ndc-rest-schema update -d ./ndc-rest-schema/command/testdata/auth + { + Argument: UpdateCommandArguments{ + Dir: "testdata/auth", + }, + Expected: "testdata/auth/expected.json", + }, } for _, tc := range testCases { diff --git a/ndc-rest-schema/configuration/schema.go b/ndc-rest-schema/configuration/schema.go index f94b0a4..22df4c6 100644 --- a/ndc-rest-schema/configuration/schema.go +++ b/ndc-rest-schema/configuration/schema.go @@ -28,7 +28,6 @@ func BuildSchemaFromConfig(config *Configuration, configDir string, logger *slog if schemaOutput == nil { continue } - ndcSchema := NDCRestRuntimeSchema{ Name: file.File, NDCRestSchema: schemaOutput, @@ -40,6 +39,7 @@ func BuildSchemaFromConfig(config *Configuration, configDir string, logger *slog } else { ndcSchema.Runtime = *runtime } + schemas[i] = ndcSchema } @@ -304,6 +304,18 @@ func buildRESTArguments(config *Configuration, restSchema *rest.NDCRestSchema, c } func buildHeadersForwardingResponse(config *Configuration, restSchema *rest.NDCRestSchema) { + if !config.ForwardHeaders.Enabled { + return + } + + if _, ok := restSchema.ScalarTypes[string(rest.ScalarJSON)]; !ok { + restSchema.ScalarTypes[string(rest.ScalarJSON)] = schema.ScalarType{ + AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, + ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, + Representation: schema.NewTypeRepresentationJSON().Encode(), + } + } + if config.ForwardHeaders.ResponseHeaders == nil { return } @@ -320,7 +332,7 @@ func buildHeadersForwardingResponse(config *Configuration, restSchema *rest.NDCR func applyOperationInfo(config *Configuration, info *rest.OperationInfo) { info.Arguments[rest.RESTOptionsArgumentName] = restSingleOptionsArgument - if config.ForwardHeaders.Enabled { + if config.ForwardHeaders.Enabled && config.ForwardHeaders.ArgumentField != nil { info.Arguments[*config.ForwardHeaders.ArgumentField] = headersArguments } } diff --git a/ndc-rest-schema/openapi/testdata/jsonplaceholder/expected.json b/ndc-rest-schema/openapi/testdata/jsonplaceholder/expected.json index 53ba330..8bdcfed 100644 --- a/ndc-rest-schema/openapi/testdata/jsonplaceholder/expected.json +++ b/ndc-rest-schema/openapi/testdata/jsonplaceholder/expected.json @@ -4,8 +4,8 @@ "servers": [ { "url": { - "env": "SERVER_URL", - "value": "https://jsonplaceholder.typicode.com" + "value": "https://jsonplaceholder.typicode.com", + "env": "SERVER_URL" } } ], @@ -34,7 +34,9 @@ "name": "id", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } }, @@ -51,13 +53,14 @@ "name": "userId", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get all available albums", - "name": "getAlbums", "result_type": { "element_type": { "name": "Album", @@ -85,13 +88,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get specific album", - "name": "getAlbumsId", "result_type": { "name": "Album", "type": "named" @@ -116,13 +120,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get photos for a specific album", - "name": "getAlbumsIdPhotos", "result_type": { "element_type": { "name": "Photo", @@ -150,13 +155,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get specific comment", - "name": "getComment", "result_type": { "name": "Comment", "type": "named" @@ -184,7 +190,9 @@ "name": "id", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } }, @@ -201,13 +209,14 @@ "name": "postId", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get all available comments", - "name": "getComments", "result_type": { "element_type": { "name": "Comment", @@ -235,13 +244,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get specific photo", - "name": "getPhoto", "result_type": { "name": "Photo", "type": "named" @@ -269,7 +279,9 @@ "name": "albumId", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } }, @@ -286,13 +298,14 @@ "name": "id", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get all available photos", - "name": "getPhotos", "result_type": { "element_type": { "name": "Photo", @@ -320,13 +333,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get specific post", - "name": "getPostById", "result_type": { "name": "Post", "type": "named" @@ -354,7 +368,9 @@ "name": "id", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } }, @@ -371,13 +387,14 @@ "name": "userId", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get all available posts", - "name": "getPosts", "result_type": { "element_type": { "name": "Post", @@ -405,13 +422,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get comments for a specific post", - "name": "getPostsIdComments", "result_type": { "element_type": { "name": "Comment", @@ -430,7 +448,6 @@ }, "arguments": {}, "description": "Get test", - "name": "getTest", "result_type": { "name": "User", "type": "named" @@ -455,13 +472,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get specific todo", - "name": "getTodo", "result_type": { "name": "Todo", "type": "named" @@ -489,7 +507,9 @@ "name": "id", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } }, @@ -506,13 +526,14 @@ "name": "userId", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get all available todos", - "name": "getTodos", "result_type": { "element_type": { "name": "Todo", @@ -540,13 +561,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get specific user", - "name": "getUser", "result_type": { "name": "User", "type": "named" @@ -574,7 +596,9 @@ "name": "email", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } }, @@ -591,13 +615,14 @@ "name": "id", "in": "query", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Get all available users", - "name": "getUsers", "result_type": { "element_type": { "name": "User", @@ -619,7 +644,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -632,7 +659,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "userId": { @@ -644,7 +673,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } } @@ -661,7 +692,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "email": { @@ -673,7 +706,9 @@ } }, "rest": { - "type": ["string"], + "type": [ + "string" + ], "format": "email" } }, @@ -686,7 +721,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -699,7 +736,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "postId": { @@ -711,7 +750,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } } @@ -728,7 +769,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -741,7 +784,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -754,7 +799,9 @@ } }, "rest": { - "type": ["string"], + "type": [ + "string" + ], "format": "uri" } }, @@ -767,7 +814,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "url": { @@ -779,7 +828,9 @@ } }, "rest": { - "type": ["string"], + "type": [ + "string" + ], "format": "uri" } } @@ -796,7 +847,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "id": { @@ -808,7 +861,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -821,7 +876,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "userId": { @@ -833,7 +890,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } } @@ -850,7 +909,9 @@ } }, "rest": { - "type": ["boolean"] + "type": [ + "boolean" + ] } }, "id": { @@ -862,7 +923,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -875,7 +938,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "userId": { @@ -887,7 +952,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } } @@ -904,7 +971,9 @@ } }, "rest": { - "type": ["object"] + "type": [ + "object" + ] } }, "company": { @@ -916,7 +985,9 @@ } }, "rest": { - "type": ["object"] + "type": [ + "object" + ] } }, "email": { @@ -928,7 +999,9 @@ } }, "rest": { - "type": ["string"], + "type": [ + "string" + ], "format": "email" } }, @@ -941,7 +1014,9 @@ } }, "rest": { - "type": ["integer"], + "type": [ + "integer" + ], "format": "int64" } }, @@ -954,7 +1029,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "phone": { @@ -966,7 +1043,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "username": { @@ -978,7 +1057,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "website": { @@ -990,7 +1071,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } } } @@ -1006,7 +1089,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "geo": { @@ -1018,7 +1103,9 @@ } }, "rest": { - "type": ["object"] + "type": [ + "object" + ] } }, "street": { @@ -1030,7 +1117,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "suite": { @@ -1042,7 +1131,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "zipcode": { @@ -1054,7 +1145,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } } } @@ -1070,7 +1163,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "lng": { @@ -1082,7 +1177,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } } } @@ -1098,7 +1195,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "catchPhrase": { @@ -1110,7 +1209,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } }, "name": { @@ -1122,7 +1223,9 @@ } }, "rest": { - "type": ["string"] + "type": [ + "string" + ] } } } @@ -1150,13 +1253,14 @@ "rest": { "in": "body", "schema": { - "type": ["object"] + "type": [ + "object" + ] } } } }, "description": "Create a post", - "name": "createPost", "result_type": { "name": "Post", "type": "named" @@ -1181,13 +1285,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Delete specific post", - "name": "deletePostById", "result_type": { "type": "nullable", "underlying_type": { @@ -1217,7 +1322,9 @@ "rest": { "in": "body", "schema": { - "type": ["object"] + "type": [ + "object" + ] } } }, @@ -1231,13 +1338,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "patch specific post", - "name": "patchPostById", "result_type": { "name": "Post", "type": "named" @@ -1264,7 +1372,9 @@ "rest": { "in": "body", "schema": { - "type": ["object"] + "type": [ + "object" + ] } } }, @@ -1278,13 +1388,14 @@ "name": "id", "in": "path", "schema": { - "type": ["integer"] + "type": [ + "integer" + ] } } } }, "description": "Update specific post", - "name": "updatePostById", "result_type": { "name": "Post", "type": "named" diff --git a/ndc-rest-schema/schema/setting.go b/ndc-rest-schema/schema/setting.go index 35beaf8..7dda824 100644 --- a/ndc-rest-schema/schema/setting.go +++ b/ndc-rest-schema/schema/setting.go @@ -79,7 +79,7 @@ type ServerConfig struct { TLS *TLSConfig `json:"tls,omitempty" mapstructure:"tls" yaml:"tls,omitempty"` // cached values that are loaded from environment variables - url *string + url *url.URL headers map[string]string } @@ -102,20 +102,21 @@ func (j *ServerConfig) UnmarshalJSON(b []byte) error { // Validate if the current instance is valid func (ss *ServerConfig) Validate() error { - urlValue, err := ss.URL.Get() + rawURL, err := ss.URL.Get() if err != nil { return fmt.Errorf("server url: %w", err) } - if urlValue == "" { + if rawURL == "" { return errors.New("url is required for server") } - if _, err := parseHttpURL(urlValue); err != nil { + urlValue, err := parseHttpURL(rawURL) + if err != nil { return fmt.Errorf("server url: %w", err) } - ss.url = &urlValue + ss.url = urlValue headers, err := getHeadersFromEnv(ss.Headers) if err != nil { @@ -127,14 +128,21 @@ func (ss *ServerConfig) Validate() error { } // Validate if the current instance is valid -func (ss ServerConfig) GetURL() string { +func (ss ServerConfig) GetURL() (url.URL, error) { if ss.url != nil { - return *ss.url + return *ss.url, nil } - result, _ := ss.URL.Get() + rawURL, err := ss.URL.Get() + if err != nil { + return url.URL{}, err + } + urlValue, err := parseHttpURL(rawURL) + if err != nil { + return url.URL{}, fmt.Errorf("server url: %w", err) + } - return result + return *urlValue, nil } // Validate if the current instance is valid diff --git a/tests/configuration/config.yaml b/tests/configuration/config.yaml new file mode 100644 index 0000000..22b1f10 --- /dev/null +++ b/tests/configuration/config.yaml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=../../ndc-rest-schema/jsonschema/configuration.schema.json +output: schema.output.json +strict: true +forwardHeaders: + enabled: false + argumentField: headers + responseHeaders: + headersField: "headers" + resultField: "response" + forwardHeaders: + - Content-Type + - X-Custom-Header +concurrency: + query: 1 + mutation: 1 + rest: 0 +files: + - file: https://raw.githubusercontent.com/hasura/ndc-rest/refs/heads/main/connector/testdata/jsonplaceholder/swagger.json + spec: oas2 diff --git a/tests/engine/.env b/tests/engine/.env new file mode 100644 index 0000000..829e46d --- /dev/null +++ b/tests/engine/.env @@ -0,0 +1,2 @@ +APP_MYAPI_READ_URL=http://local.hasura.dev:8080 +APP_MYAPI_WRITE_URL=http://local.hasura.dev:8080 \ No newline at end of file diff --git a/tests/engine/.gitattributes b/tests/engine/.gitattributes new file mode 100644 index 0000000..8ddc99f --- /dev/null +++ b/tests/engine/.gitattributes @@ -0,0 +1 @@ +*.hml linguist-language=yaml \ No newline at end of file diff --git a/tests/engine/.gitignore b/tests/engine/.gitignore new file mode 100644 index 0000000..d168928 --- /dev/null +++ b/tests/engine/.gitignore @@ -0,0 +1,2 @@ +engine/build +/.env.* diff --git a/tests/engine/.hasura/context.yaml b/tests/engine/.hasura/context.yaml new file mode 100644 index 0000000..3822ed0 --- /dev/null +++ b/tests/engine/.hasura/context.yaml @@ -0,0 +1,14 @@ +kind: Context +version: v3 +definition: + current: default + contexts: + default: + supergraph: ../supergraph.yaml + subgraph: ../app/subgraph.yaml + localEnvFile: ../.env + scripts: + docker-start: + bash: HASURA_DDN_PAT=$(ddn auth print-pat) PROMPTQL_SECRET_KEY=$(ddn auth print-promptql-secret-key) docker compose -f compose.yaml --env-file .env up --build --pull always + powershell: $Env:HASURA_DDN_PAT = ddn auth print-pat; $Env:PROMPTQL_SECRET_KEY = ddn auth print-promptql-secret-key; docker compose -f compose.yaml --env-file .env up --build --pull always + promptQL: false diff --git a/tests/engine/app/metadata/createPost.hml b/tests/engine/app/metadata/createPost.hml new file mode 100644 index 0000000..d140526 --- /dev/null +++ b/tests/engine/app/metadata/createPost.hml @@ -0,0 +1,28 @@ +--- +kind: Command +version: v1 +definition: + name: createPost + outputType: Post! + arguments: + - name: body + type: Post! + description: Post object that needs to be added + source: + dataConnectorName: myapi + dataConnectorCommand: + procedure: createPost + graphql: + rootFieldName: createPost + rootFieldKind: Mutation + description: Create a post + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: createPost + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/deletePostById.hml b/tests/engine/app/metadata/deletePostById.hml new file mode 100644 index 0000000..dbc7139 --- /dev/null +++ b/tests/engine/app/metadata/deletePostById.hml @@ -0,0 +1,28 @@ +--- +kind: Command +version: v1 +definition: + name: deletePostById + outputType: Boolean + arguments: + - name: id + type: Int32! + description: The ID of the post to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + procedure: deletePostById + graphql: + rootFieldName: deletePostById + rootFieldKind: Mutation + description: Delete specific post + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: deletePostById + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getAlbums.hml b/tests/engine/app/metadata/getAlbums.hml new file mode 100644 index 0000000..0c2a1ab --- /dev/null +++ b/tests/engine/app/metadata/getAlbums.hml @@ -0,0 +1,63 @@ +--- +kind: ObjectType +version: v1 +definition: + name: Album + fields: + - name: id + type: Int64 + - name: title + type: String + - name: userId + type: Int64 + graphql: + typeName: Album + inputTypeName: Album_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: Album + +--- +kind: TypePermissions +version: v1 +definition: + typeName: Album + permissions: + - role: admin + output: + allowedFields: + - id + - title + - userId + +--- +kind: Command +version: v1 +definition: + name: getAlbums + outputType: "[Album!]!" + arguments: + - name: id + type: Int32 + description: Filter by album ID + - name: userId + type: Int32 + description: Filter by user ID + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getAlbums + graphql: + rootFieldName: getAlbums + rootFieldKind: Query + description: Get all available albums + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getAlbums + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getAlbumsId.hml b/tests/engine/app/metadata/getAlbumsId.hml new file mode 100644 index 0000000..b09cd7a --- /dev/null +++ b/tests/engine/app/metadata/getAlbumsId.hml @@ -0,0 +1,28 @@ +--- +kind: Command +version: v1 +definition: + name: getAlbumsId + outputType: Album! + arguments: + - name: id + type: Int32! + description: The ID of the album to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getAlbumsId + graphql: + rootFieldName: getAlbumsId + rootFieldKind: Query + description: Get specific album + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getAlbumsId + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getAlbumsIdPhotos.hml b/tests/engine/app/metadata/getAlbumsIdPhotos.hml new file mode 100644 index 0000000..76a9381 --- /dev/null +++ b/tests/engine/app/metadata/getAlbumsIdPhotos.hml @@ -0,0 +1,66 @@ +--- +kind: ObjectType +version: v1 +definition: + name: Photo + fields: + - name: albumId + type: Int64 + - name: id + type: Int64 + - name: thumbnailUrl + type: URI + - name: title + type: String + - name: url + type: URI + graphql: + typeName: Photo + inputTypeName: Photo_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: Photo + +--- +kind: TypePermissions +version: v1 +definition: + typeName: Photo + permissions: + - role: admin + output: + allowedFields: + - albumId + - id + - thumbnailUrl + - title + - url + +--- +kind: Command +version: v1 +definition: + name: getAlbumsIdPhotos + outputType: "[Photo!]!" + arguments: + - name: id + type: Int32! + description: post id + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getAlbumsIdPhotos + graphql: + rootFieldName: getAlbumsIdPhotos + rootFieldKind: Query + description: Get photos for a specific album + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getAlbumsIdPhotos + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getComment.hml b/tests/engine/app/metadata/getComment.hml new file mode 100644 index 0000000..4c7fa24 --- /dev/null +++ b/tests/engine/app/metadata/getComment.hml @@ -0,0 +1,66 @@ +--- +kind: ObjectType +version: v1 +definition: + name: Comment + fields: + - name: body + type: String + - name: email + type: String + - name: id + type: Int64 + - name: name + type: String + - name: postId + type: Int64 + graphql: + typeName: Comment + inputTypeName: Comment_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: Comment + +--- +kind: TypePermissions +version: v1 +definition: + typeName: Comment + permissions: + - role: admin + output: + allowedFields: + - body + - email + - id + - name + - postId + +--- +kind: Command +version: v1 +definition: + name: getComment + outputType: Comment! + arguments: + - name: id + type: Int32! + description: The ID of the comment to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getComment + graphql: + rootFieldName: getComment + rootFieldKind: Query + description: Get specific comment + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getComment + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getComments.hml b/tests/engine/app/metadata/getComments.hml new file mode 100644 index 0000000..860d267 --- /dev/null +++ b/tests/engine/app/metadata/getComments.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: getComments + outputType: "[Comment!]!" + arguments: + - name: id + type: Int32 + description: Filter by comment ID + - name: postId + type: Int32 + description: Filter by post ID + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getComments + graphql: + rootFieldName: getComments + rootFieldKind: Query + description: Get all available comments + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getComments + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getPhoto.hml b/tests/engine/app/metadata/getPhoto.hml new file mode 100644 index 0000000..b04c2df --- /dev/null +++ b/tests/engine/app/metadata/getPhoto.hml @@ -0,0 +1,28 @@ +--- +kind: Command +version: v1 +definition: + name: getPhoto + outputType: Photo! + arguments: + - name: id + type: Int32! + description: The ID of the photo to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getPhoto + graphql: + rootFieldName: getPhoto + rootFieldKind: Query + description: Get specific photo + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getPhoto + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getPhotos.hml b/tests/engine/app/metadata/getPhotos.hml new file mode 100644 index 0000000..5a28de4 --- /dev/null +++ b/tests/engine/app/metadata/getPhotos.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: getPhotos + outputType: "[Photo!]!" + arguments: + - name: albumId + type: Int32 + description: Filter by album ID + - name: id + type: Int32 + description: Filter by photo ID + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getPhotos + graphql: + rootFieldName: getPhotos + rootFieldKind: Query + description: Get all available photos + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getPhotos + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getPostById.hml b/tests/engine/app/metadata/getPostById.hml new file mode 100644 index 0000000..8813e67 --- /dev/null +++ b/tests/engine/app/metadata/getPostById.hml @@ -0,0 +1,63 @@ +--- +kind: ObjectType +version: v1 +definition: + name: Post + fields: + - name: body + type: String + - name: id + type: Int64 + - name: title + type: String + - name: userId + type: Int64 + graphql: + typeName: Post + inputTypeName: Post_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: Post + +--- +kind: TypePermissions +version: v1 +definition: + typeName: Post + permissions: + - role: admin + output: + allowedFields: + - body + - id + - title + - userId + +--- +kind: Command +version: v1 +definition: + name: getPostById + outputType: Post! + arguments: + - name: id + type: Int32! + description: The ID of the post to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getPostById + graphql: + rootFieldName: getPostById + rootFieldKind: Query + description: Get specific post + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getPostById + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getPosts.hml b/tests/engine/app/metadata/getPosts.hml new file mode 100644 index 0000000..1f1e288 --- /dev/null +++ b/tests/engine/app/metadata/getPosts.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: getPosts + outputType: "[Post!]!" + arguments: + - name: id + type: Int32 + description: Filter by post ID + - name: userId + type: Int32 + description: Filter by user ID + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getPosts + graphql: + rootFieldName: getPosts + rootFieldKind: Query + description: Get all available posts + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getPosts + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getPostsIdComments.hml b/tests/engine/app/metadata/getPostsIdComments.hml new file mode 100644 index 0000000..4a8aeb9 --- /dev/null +++ b/tests/engine/app/metadata/getPostsIdComments.hml @@ -0,0 +1,28 @@ +--- +kind: Command +version: v1 +definition: + name: getPostsIdComments + outputType: "[Comment!]!" + arguments: + - name: id + type: Int32! + description: post id + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getPostsIdComments + graphql: + rootFieldName: getPostsIdComments + rootFieldKind: Query + description: Get comments for a specific post + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getPostsIdComments + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getTodo.hml b/tests/engine/app/metadata/getTodo.hml new file mode 100644 index 0000000..01cfdc1 --- /dev/null +++ b/tests/engine/app/metadata/getTodo.hml @@ -0,0 +1,63 @@ +--- +kind: ObjectType +version: v1 +definition: + name: Todo + fields: + - name: completed + type: Boolean + - name: id + type: Int64 + - name: title + type: String + - name: userId + type: Int64 + graphql: + typeName: Todo + inputTypeName: Todo_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: Todo + +--- +kind: TypePermissions +version: v1 +definition: + typeName: Todo + permissions: + - role: admin + output: + allowedFields: + - completed + - id + - title + - userId + +--- +kind: Command +version: v1 +definition: + name: getTodo + outputType: Todo! + arguments: + - name: id + type: Int32! + description: The ID of the todo to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getTodo + graphql: + rootFieldName: getTodo + rootFieldKind: Query + description: Get specific todo + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getTodo + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getTodos.hml b/tests/engine/app/metadata/getTodos.hml new file mode 100644 index 0000000..050e1b9 --- /dev/null +++ b/tests/engine/app/metadata/getTodos.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: getTodos + outputType: "[Todo!]!" + arguments: + - name: id + type: Int32 + description: Filter by todo ID + - name: userId + type: Int32 + description: Filter by user ID + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getTodos + graphql: + rootFieldName: getTodos + rootFieldKind: Query + description: Get all available todos + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getTodos + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getUser.hml b/tests/engine/app/metadata/getUser.hml new file mode 100644 index 0000000..b1022de --- /dev/null +++ b/tests/engine/app/metadata/getUser.hml @@ -0,0 +1,174 @@ +--- +kind: ObjectType +version: v1 +definition: + name: UserAddressGeo + fields: + - name: lat + type: String + - name: lng + type: String + graphql: + typeName: UserAddressGeo + inputTypeName: UserAddressGeo_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: UserAddressGeo + +--- +kind: TypePermissions +version: v1 +definition: + typeName: UserAddressGeo + permissions: + - role: admin + output: + allowedFields: + - lat + - lng + +--- +kind: ObjectType +version: v1 +definition: + name: UserAddress + fields: + - name: city + type: String + - name: geo + type: UserAddressGeo + - name: street + type: String + - name: suite + type: String + - name: zipcode + type: String + graphql: + typeName: UserAddress + inputTypeName: UserAddress_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: UserAddress + +--- +kind: TypePermissions +version: v1 +definition: + typeName: UserAddress + permissions: + - role: admin + output: + allowedFields: + - city + - geo + - street + - suite + - zipcode + +--- +kind: ObjectType +version: v1 +definition: + name: UserCompany + fields: + - name: bs + type: String + - name: catchPhrase + type: String + - name: name + type: String + graphql: + typeName: UserCompany + inputTypeName: UserCompany_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: UserCompany + +--- +kind: TypePermissions +version: v1 +definition: + typeName: UserCompany + permissions: + - role: admin + output: + allowedFields: + - bs + - catchPhrase + - name + +--- +kind: ObjectType +version: v1 +definition: + name: User + fields: + - name: address + type: UserAddress + - name: company + type: UserCompany + - name: email + type: String + - name: id + type: Int64 + - name: name + type: String + - name: phone + type: String + - name: username + type: String + - name: website + type: String + graphql: + typeName: User + inputTypeName: User_input + dataConnectorTypeMapping: + - dataConnectorName: myapi + dataConnectorObjectType: User + +--- +kind: TypePermissions +version: v1 +definition: + typeName: User + permissions: + - role: admin + output: + allowedFields: + - address + - company + - email + - id + - name + - phone + - username + - website + +--- +kind: Command +version: v1 +definition: + name: getUser + outputType: User! + arguments: + - name: id + type: Int32! + description: The ID of the user to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getUser + graphql: + rootFieldName: getUser + rootFieldKind: Query + description: Get specific user + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getUser + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/getUsers.hml b/tests/engine/app/metadata/getUsers.hml new file mode 100644 index 0000000..b1f3c32 --- /dev/null +++ b/tests/engine/app/metadata/getUsers.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: getUsers + outputType: "[User!]!" + arguments: + - name: email + type: Int32 + description: Filter by user email address + - name: id + type: Int32 + description: Filter by user ID + source: + dataConnectorName: myapi + dataConnectorCommand: + function: getUsers + graphql: + rootFieldName: getUsers + rootFieldKind: Query + description: Get all available users + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: getUsers + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/myapi-types.hml b/tests/engine/app/metadata/myapi-types.hml new file mode 100644 index 0000000..e46c3bc --- /dev/null +++ b/tests/engine/app/metadata/myapi-types.hml @@ -0,0 +1,114 @@ +--- +kind: ScalarType +version: v1 +definition: + name: Int32 + graphql: + typeName: Int32 + +--- +kind: BooleanExpressionType +version: v1 +definition: + name: Int32_bool_exp + operand: + scalar: + type: Int32 + comparisonOperators: [] + dataConnectorOperatorMapping: + - dataConnectorName: myapi + dataConnectorScalarType: Int32 + operatorMapping: {} + logicalOperators: + enable: true + isNull: + enable: true + graphql: + typeName: Int32_bool_exp + +--- +kind: DataConnectorScalarRepresentation +version: v1 +definition: + dataConnectorName: myapi + dataConnectorScalarType: Int32 + representation: Int32 + graphql: + comparisonExpressionTypeName: Int32_comparison_exp + +--- +kind: ScalarType +version: v1 +definition: + name: Int64 + graphql: + typeName: Int64 + +--- +kind: BooleanExpressionType +version: v1 +definition: + name: Int64_bool_exp + operand: + scalar: + type: Int64 + comparisonOperators: [] + dataConnectorOperatorMapping: + - dataConnectorName: myapi + dataConnectorScalarType: Int64 + operatorMapping: {} + logicalOperators: + enable: true + isNull: + enable: true + graphql: + typeName: Int64_bool_exp + +--- +kind: DataConnectorScalarRepresentation +version: v1 +definition: + dataConnectorName: myapi + dataConnectorScalarType: Int64 + representation: Int64 + graphql: + comparisonExpressionTypeName: Int64_comparison_exp + +--- +kind: ScalarType +version: v1 +definition: + name: URI + graphql: + typeName: URI + +--- +kind: BooleanExpressionType +version: v1 +definition: + name: URI_bool_exp + operand: + scalar: + type: URI + comparisonOperators: [] + dataConnectorOperatorMapping: + - dataConnectorName: myapi + dataConnectorScalarType: URI + operatorMapping: {} + logicalOperators: + enable: true + isNull: + enable: true + graphql: + typeName: URI_bool_exp + +--- +kind: DataConnectorScalarRepresentation +version: v1 +definition: + dataConnectorName: myapi + dataConnectorScalarType: URI + representation: URI + graphql: + comparisonExpressionTypeName: URI_comparison_exp + diff --git a/tests/engine/app/metadata/myapi.hml b/tests/engine/app/metadata/myapi.hml new file mode 100644 index 0000000..9602495 --- /dev/null +++ b/tests/engine/app/metadata/myapi.hml @@ -0,0 +1,588 @@ +kind: DataConnectorLink +version: v1 +definition: + name: myapi + url: + readWriteUrls: + read: + valueFromEnv: APP_MYAPI_READ_URL + write: + valueFromEnv: APP_MYAPI_WRITE_URL + headers: + X-Test-Header: + value: test + schema: + version: v0.1 + schema: + scalar_types: + Boolean: + representation: + type: boolean + aggregate_functions: {} + comparison_operators: {} + Int32: + representation: + type: int32 + aggregate_functions: {} + comparison_operators: {} + Int64: + representation: + type: int64 + aggregate_functions: {} + comparison_operators: {} + String: + representation: + type: string + aggregate_functions: {} + comparison_operators: {} + URI: + representation: + type: string + aggregate_functions: {} + comparison_operators: {} + object_types: + Album: + fields: + id: + type: + type: nullable + underlying_type: + type: named + name: Int64 + title: + type: + type: nullable + underlying_type: + type: named + name: String + userId: + type: + type: nullable + underlying_type: + type: named + name: Int64 + Comment: + fields: + body: + type: + type: nullable + underlying_type: + type: named + name: String + email: + type: + type: nullable + underlying_type: + type: named + name: String + id: + type: + type: nullable + underlying_type: + type: named + name: Int64 + name: + type: + type: nullable + underlying_type: + type: named + name: String + postId: + type: + type: nullable + underlying_type: + type: named + name: Int64 + Photo: + fields: + albumId: + type: + type: nullable + underlying_type: + type: named + name: Int64 + id: + type: + type: nullable + underlying_type: + type: named + name: Int64 + thumbnailUrl: + type: + type: nullable + underlying_type: + type: named + name: URI + title: + type: + type: nullable + underlying_type: + type: named + name: String + url: + type: + type: nullable + underlying_type: + type: named + name: URI + Post: + fields: + body: + type: + type: nullable + underlying_type: + type: named + name: String + id: + type: + type: nullable + underlying_type: + type: named + name: Int64 + title: + type: + type: nullable + underlying_type: + type: named + name: String + userId: + type: + type: nullable + underlying_type: + type: named + name: Int64 + Todo: + fields: + completed: + type: + type: nullable + underlying_type: + type: named + name: Boolean + id: + type: + type: nullable + underlying_type: + type: named + name: Int64 + title: + type: + type: nullable + underlying_type: + type: named + name: String + userId: + type: + type: nullable + underlying_type: + type: named + name: Int64 + User: + fields: + address: + type: + type: nullable + underlying_type: + type: named + name: UserAddress + company: + type: + type: nullable + underlying_type: + type: named + name: UserCompany + email: + type: + type: nullable + underlying_type: + type: named + name: String + id: + type: + type: nullable + underlying_type: + type: named + name: Int64 + name: + type: + type: nullable + underlying_type: + type: named + name: String + phone: + type: + type: nullable + underlying_type: + type: named + name: String + username: + type: + type: nullable + underlying_type: + type: named + name: String + website: + type: + type: nullable + underlying_type: + type: named + name: String + UserAddress: + fields: + city: + type: + type: nullable + underlying_type: + type: named + name: String + geo: + type: + type: nullable + underlying_type: + type: named + name: UserAddressGeo + street: + type: + type: nullable + underlying_type: + type: named + name: String + suite: + type: + type: nullable + underlying_type: + type: named + name: String + zipcode: + type: + type: nullable + underlying_type: + type: named + name: String + UserAddressGeo: + fields: + lat: + type: + type: nullable + underlying_type: + type: named + name: String + lng: + type: + type: nullable + underlying_type: + type: named + name: String + UserCompany: + fields: + bs: + type: + type: nullable + underlying_type: + type: named + name: String + catchPhrase: + type: + type: nullable + underlying_type: + type: named + name: String + name: + type: + type: nullable + underlying_type: + type: named + name: String + collections: [] + functions: + - name: getAlbums + description: Get all available albums + arguments: + id: + description: Filter by album ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + userId: + description: Filter by user ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Album + - name: getAlbumsId + description: Get specific album + arguments: + id: + description: The ID of the album to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Album + - name: getAlbumsIdPhotos + description: Get photos for a specific album + arguments: + id: + description: post id + type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Photo + - name: getComment + description: Get specific comment + arguments: + id: + description: The ID of the comment to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Comment + - name: getComments + description: Get all available comments + arguments: + id: + description: Filter by comment ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + postId: + description: Filter by post ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Comment + - name: getPhoto + description: Get specific photo + arguments: + id: + description: The ID of the photo to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Photo + - name: getPhotos + description: Get all available photos + arguments: + albumId: + description: Filter by album ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + id: + description: Filter by photo ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Photo + - name: getPostById + description: Get specific post + arguments: + id: + description: The ID of the post to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Post + - name: getPosts + description: Get all available posts + arguments: + id: + description: Filter by post ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + userId: + description: Filter by user ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Post + - name: getPostsIdComments + description: Get comments for a specific post + arguments: + id: + description: post id + type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Comment + - name: getTodo + description: Get specific todo + arguments: + id: + description: The ID of the todo to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Todo + - name: getTodos + description: Get all available todos + arguments: + id: + description: Filter by todo ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + userId: + description: Filter by user ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: Todo + - name: getUser + description: Get specific user + arguments: + id: + description: The ID of the user to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: User + - name: getUsers + description: Get all available users + arguments: + email: + description: Filter by user email address + type: + type: nullable + underlying_type: + type: named + name: Int32 + id: + description: Filter by user ID + type: + type: nullable + underlying_type: + type: named + name: Int32 + result_type: + type: array + element_type: + type: named + name: User + procedures: + - name: createPost + description: Create a post + arguments: + body: + description: Post object that needs to be added + type: + type: named + name: Post + result_type: + type: named + name: Post + - name: deletePostById + description: Delete specific post + arguments: + id: + description: The ID of the post to retrieve + type: + type: named + name: Int32 + result_type: + type: nullable + underlying_type: + type: named + name: Boolean + - name: patchPostById + description: patch specific post + arguments: + body: + description: Post object that needs to be updated + type: + type: named + name: Post + id: + description: The ID of the post to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Post + - name: updatePostById + description: Update specific post + arguments: + body: + description: Post object that needs to be updated + type: + type: named + name: Post + id: + description: The ID of the post to retrieve + type: + type: named + name: Int32 + result_type: + type: named + name: Post + capabilities: + version: 0.1.6 + capabilities: + query: + variables: {} + explain: {} + nested_fields: {} + exists: {} + mutation: + explain: {} diff --git a/tests/engine/app/metadata/patchPostById.hml b/tests/engine/app/metadata/patchPostById.hml new file mode 100644 index 0000000..6d957d1 --- /dev/null +++ b/tests/engine/app/metadata/patchPostById.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: patchPostById + outputType: Post! + arguments: + - name: body + type: Post! + description: Post object that needs to be updated + - name: id + type: Int32! + description: The ID of the post to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + procedure: patchPostById + graphql: + rootFieldName: patchPostById + rootFieldKind: Mutation + description: patch specific post + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: patchPostById + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/updatePostById.hml b/tests/engine/app/metadata/updatePostById.hml new file mode 100644 index 0000000..50c6c57 --- /dev/null +++ b/tests/engine/app/metadata/updatePostById.hml @@ -0,0 +1,31 @@ +--- +kind: Command +version: v1 +definition: + name: updatePostById + outputType: Post! + arguments: + - name: body + type: Post! + description: Post object that needs to be updated + - name: id + type: Int32! + description: The ID of the post to retrieve + source: + dataConnectorName: myapi + dataConnectorCommand: + procedure: updatePostById + graphql: + rootFieldName: updatePostById + rootFieldKind: Mutation + description: Update specific post + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: updatePostById + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/subgraph.yaml b/tests/engine/app/subgraph.yaml new file mode 100644 index 0000000..1d7ebb0 --- /dev/null +++ b/tests/engine/app/subgraph.yaml @@ -0,0 +1,14 @@ +kind: Subgraph +version: v2 +definition: + name: app + generator: + rootPath: . + namingConvention: none + includePaths: + - metadata + envMapping: + APP_MYAPI_READ_URL: + fromEnv: APP_MYAPI_READ_URL + APP_MYAPI_WRITE_URL: + fromEnv: APP_MYAPI_WRITE_URL diff --git a/tests/engine/compose.yaml b/tests/engine/compose.yaml new file mode 100644 index 0000000..b9cd1ec --- /dev/null +++ b/tests/engine/compose.yaml @@ -0,0 +1,30 @@ +services: + engine: + build: + context: engine + dockerfile: Dockerfile.engine + pull: true + environment: + AUTHN_CONFIG_PATH: /md/auth_config.json + ENABLE_CORS: "true" + ENABLE_SQL_INTERFACE: "true" + INTROSPECTION_METADATA_FILE: /md/metadata.json + METADATA_PATH: /md/open_dd.json + OTLP_ENDPOINT: http://local.hasura.dev:4317 + extra_hosts: + - local.hasura.dev:host-gateway + labels: + io.hasura.ddn.service-name: engine + ports: + - 3280:3000 + otel-collector: + command: + - --config=/etc/otel-collector-config.yaml + environment: + HASURA_DDN_PAT: ${HASURA_DDN_PAT} + image: otel/opentelemetry-collector:0.104.0 + ports: + - 4317:4317 + - 4318:4318 + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml diff --git a/tests/engine/engine/Dockerfile.engine b/tests/engine/engine/Dockerfile.engine new file mode 100644 index 0000000..3613f0e --- /dev/null +++ b/tests/engine/engine/Dockerfile.engine @@ -0,0 +1,2 @@ +FROM ghcr.io/hasura/v3-engine +COPY ./build /md/ \ No newline at end of file diff --git a/tests/engine/globals/metadata/auth-config.hml b/tests/engine/globals/metadata/auth-config.hml new file mode 100644 index 0000000..62c27d7 --- /dev/null +++ b/tests/engine/globals/metadata/auth-config.hml @@ -0,0 +1,8 @@ +kind: AuthConfig +version: v2 +definition: + mode: + noAuth: + role: admin + sessionVariables: + X-Custom-Header: foo diff --git a/tests/engine/globals/metadata/compatibility-config.hml b/tests/engine/globals/metadata/compatibility-config.hml new file mode 100644 index 0000000..10d0471 --- /dev/null +++ b/tests/engine/globals/metadata/compatibility-config.hml @@ -0,0 +1,2 @@ +kind: CompatibilityConfig +date: "2024-10-01" diff --git a/tests/engine/globals/metadata/graphql-config.hml b/tests/engine/globals/metadata/graphql-config.hml new file mode 100644 index 0000000..f938ad0 --- /dev/null +++ b/tests/engine/globals/metadata/graphql-config.hml @@ -0,0 +1,34 @@ +kind: GraphqlConfig +version: v1 +definition: + query: + rootOperationTypeName: Query + argumentsInput: + fieldName: args + limitInput: + fieldName: limit + offsetInput: + fieldName: offset + filterInput: + fieldName: where + operatorNames: + and: _and + or: _or + not: _not + isNull: _is_null + orderByInput: + fieldName: order_by + enumDirectionValues: + asc: Asc + desc: Desc + enumTypeNames: + - directions: + - Asc + - Desc + typeName: OrderBy + aggregate: + filterInputFieldName: filter_input + countFieldName: _count + countDistinctFieldName: _count_distinct + mutation: + rootOperationTypeName: Mutation diff --git a/tests/engine/globals/subgraph.yaml b/tests/engine/globals/subgraph.yaml new file mode 100644 index 0000000..b21faca --- /dev/null +++ b/tests/engine/globals/subgraph.yaml @@ -0,0 +1,8 @@ +kind: Subgraph +version: v2 +definition: + name: globals + generator: + rootPath: . + includePaths: + - metadata diff --git a/tests/engine/hasura.yaml b/tests/engine/hasura.yaml new file mode 100644 index 0000000..7f8f5cc --- /dev/null +++ b/tests/engine/hasura.yaml @@ -0,0 +1 @@ +version: v3 diff --git a/tests/engine/otel-collector-config.yaml b/tests/engine/otel-collector-config.yaml new file mode 100644 index 0000000..e4bcd1e --- /dev/null +++ b/tests/engine/otel-collector-config.yaml @@ -0,0 +1,37 @@ +exporters: + otlp: + endpoint: https://gateway.otlp.hasura.io:443 + headers: + Authorization: pat ${env:HASURA_DDN_PAT} +processors: + batch: {} +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +service: + pipelines: + logs: + exporters: + - otlp + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - otlp + processors: + - batch + receivers: + - otlp + traces: + exporters: + - otlp + processors: + - batch + receivers: + - otlp diff --git a/tests/engine/supergraph.yaml b/tests/engine/supergraph.yaml new file mode 100644 index 0000000..0d9260e --- /dev/null +++ b/tests/engine/supergraph.yaml @@ -0,0 +1,6 @@ +kind: Supergraph +version: v2 +definition: + subgraphs: + - globals/subgraph.yaml + - app/subgraph.yaml