feat: cli plugin (#27)

.github/workflows/release.yaml

@@ -2,7 +2,7 @@ name: Release container definition
 on:
   push:
     tags:
-      - "*"
+      - "v*"
 
 env:
   DOCKER_REGISTRY: ghcr.io

.gitignore

@@ -27,4 +27,5 @@ go.work.sum
 
 # Release directory
 release/
-_output/
+_output/
+schema.output.json

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "yaml.schemas": {
+    "file:///home/hgiasac/.vscode/extensions/docsmsft.docs-yaml-1.0.4/dist/toc.schema.json": "/toc\\.yml/i",
+    "file:///home/hgiasac/projects/hasura/src/ndc-rest/ndc-rest-schema/jsonschema/configuration.schema.json": "file:///home/hgiasac/projects/hasura/src/ndc-rest/ndc-rest-schema/command/testdata/auth/config.yaml"
+  }
+}

Makefile

@@ -45,4 +45,24 @@ ci-build-cli: clean
 	go run github.com/mitchellh/gox -ldflags '-X github.com/hasura/ndc-rest/ndc-rest-schema/version.BuildVersion=$(VERSION) -s -w -extldflags "-static"' \
 		-osarch="linux/amd64 darwin/amd64 windows/amd64 darwin/arm64 linux/arm64" \
 		-output="../$(OUTPUT_DIR)/ndc-rest-schema-{{.OS}}-{{.Arch}}" \
-		.
+		.
+
+.PHONY: generate-test-config
+generate-test-config:
+	go run ./ndc-rest-schema update -d ./tests/configuration
+
+.PHONY: start-ddn
+start-ddn:
+	HASURA_DDN_PAT=$$(ddn auth print-pat) docker compose --env-file tests/engine/.env up --build -d
+
+.PHONY: stop-ddn
+stop-ddn:
+	docker compose down --remove-orphans
+
+.PHONY: build-supergraph-test
+build-supergraph-test:
+	docker compose up -d --build ndc-rest
+	cd tests/engine && \
+		ddn connector-link update myapi --add-all-resources --subgraph ./app/subgraph.yaml && \
+		ddn supergraph build local
+	make start-ddn

README.md

@@ -39,6 +39,8 @@ files:
 
 The config of each element follows the [config schema](https://github.com/hasura/ndc-rest/ndc-rest-schema/blob/main/config.example.yaml) of `ndc-rest-schema`.
 
+You can add many OpenAPI files into the same connector.
+
 > [!IMPORTANT]
 > Conflicted object and scalar types will be ignored. Only the type of the first file is kept in the schema.
 
@@ -64,29 +66,67 @@ ndc-rest-schema convert -f ./rest/testdata/jsonplaceholder/swagger.json -o ./res
 - `text/*`
 - Upload file content types, e.g.`image/*` from `base64` arguments.
 
-### Environment variable template
+### Authentication
 
-The connector can replaces `{{xxx}}` templates with environment variables. The converter automatically renders variables for API keys and tokens when converting OpenAPI documents. However, you can add your custom variables as well.
+The current version supports API key and Auth token authentication schemes. The configuration is inspired from `securitySchemes` [with env variables](https://github.com/hasura/ndc-rest/ndc-rest-schema#authentication). The connector supports the following authentication strategies:
 
-### Timeout and retry
+- API Key
+- Bearer Auth
+- Cookies
+- OAuth 2.0
+
+The configuration automatically generates environment variables for the api key and Bearer token.
+
+For Cookie authentication and OAuth 2.0, you need to enable headers forwarding from the Hasura engine to the connector.
 
-The global timeout and retry strategy can be configured in the `settings` object.
+### Header Forwarding
+
+Enable `forwardHeaders` in the configuration file.
 
 ```yaml
-settings:
-  timeout: 30
-  retry:
-    times: 2
-    # delay between each retry in milliseconds
-    delay: 1000
-    httpStatus: [429, 500, 502, 503]
+# ...
+forwardHeaders:
+  enabled: true
+  argumentField: headers
 ```
 
-### Authentication
+And configure in the connector link metadata.
+
+```yaml
+kind: DataConnectorLink
+version: v1
+definition:
+  name: my_api
+  # ...
+  argumentPresets:
+    - argument: headers
+      value:
+        httpHeaders:
+          forward:
+            - Cookie
+          additional: {}
+```
+
+See the configuration example in [Hasura docs](https://hasura.io/docs/3.0/recipes/business-logic/http-header-forwarding/#step-2-update-the-metadata-1).
+
+### Timeout and retry
 
-The current version supports API key and Auth token authentication schemes. The configuration is inspired from `securitySchemes` [with env variables](https://github.com/hasura/ndc-rest/ndc-rest-schema#authentication)
+The global timeout and retry strategy can be configured in each file:
 
-See [this example](rest/testdata/auth/schema.yaml) for more context.
+```yaml
+files:
+  - file: swagger.json
+    spec: oas2
+    timeout:
+      value: 30
+    retry:
+      times:
+        value: 1
+      delay:
+        # delay between each retry in milliseconds
+        value: 500
+      httpStatus: [429, 500, 502, 503]
+```
 
 ## Distributed execution
 

compose.yaml

@@ -0,0 +1,15 @@
+include:
+  - tests/engine/compose.yaml
+services:
+  ndc-rest:
+    build:
+      context: .
+    ports:
+      - 8080:8080
+    volumes:
+      - ./tests/configuration:/etc/connector:ro
+    extra_hosts:
+      - local.hasura.dev=host-gateway
+    environment:
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://local.hasura.dev:4317
+      HASURA_LOG_LEVEL: debug

connector-definition/config.yaml

@@ -1,3 +1,21 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/hasura/ndc-rest-schema/main/jsonschema/configuration.schema.json
+output: schema.output.json
+strict: true
+forwardHeaders:
+  enabled: false
+  argumentField: headers
+  responseHeaders: null
+concurrency:
+  query: 1
+  mutation: 1
+  rest: 5
 files:
   - file: https://raw.githubusercontent.com/hasura/ndc-rest/main/connector/testdata/jsonplaceholder/swagger.json
-    spec: openapi2
+    spec: oas2
+    timeout:
+      value: 30
+    retry:
+      times:
+        value: 1
+      delay:
+        value: 500

connector/cli.go

@@ -1,44 +1,10 @@
 package rest
 
 import (
-	"context"
-	"log/slog"
-	"os"
-	"strings"
-
-	"github.com/hasura/ndc-rest/ndc-rest-schema/command"
 	"github.com/hasura/ndc-sdk-go/connector"
-	"github.com/lmittmann/tint"
 )
 
-// CLI extends the NDC SDK with custom commands
-type CLI struct {
-	connector.ServeCLI
-	Convert command.ConvertCommandArguments `cmd:"" help:"Convert API spec to NDC schema. For example:\n ndc-rest-schema convert -f petstore.yaml -o petstore.json"`
-}
-
-// Execute executes custom commands
-func (c CLI) Execute(ctx context.Context, cmd string) error {
-	switch cmd {
-	case "convert":
-		var logLevel slog.Level
-		err := logLevel.UnmarshalText([]byte(strings.ToUpper(c.LogLevel)))
-		if err != nil {
-			return err
-		}
-		logger := slog.New(tint.NewHandler(os.Stderr, &tint.Options{
-			Level:      logLevel,
-			TimeFormat: "15:04",
-		}))
-
-		return command.CommandConvertToNDCSchema(&c.Convert, logger)
-	default:
-		return c.ServeCLI.Execute(ctx, cmd)
-	}
-}
-
-// Start wrap the connector.Start function with custom CLI
+// Start and serve the connector API server
 func Start[Configuration, State any](restConnector connector.Connector[Configuration, State], options ...connector.ServeOption) error {
-	var cli CLI
-	return connector.StartCustom[Configuration, State](&cli, restConnector, options...)
+	return connector.Start(restConnector, options...)
 }

connector/connector.go

@@ -4,18 +4,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
 
 	"github.com/hasura/ndc-rest/connector/internal"
 	"github.com/hasura/ndc-rest/ndc-rest-schema/configuration"
 	rest "github.com/hasura/ndc-rest/ndc-rest-schema/schema"
 	"github.com/hasura/ndc-sdk-go/connector"
 	"github.com/hasura/ndc-sdk-go/schema"
-	"gopkg.in/yaml.v3"
 )
 
 // RESTConnector implements the SDK interface of NDC specification
 type RESTConnector struct {
+	config       *configuration.Configuration
 	metadata     internal.MetadataCollection
 	capabilities *schema.RawCapabilitiesResponse
 	rawSchema    *schema.RawSchemaResponse
@@ -43,8 +42,11 @@ func (c *RESTConnector) ParseConfiguration(ctx context.Context, configurationDir
 			Query: schema.QueryCapabilities{
 				Variables:    schema.LeafCapability{},
 				NestedFields: schema.NestedFieldCapabilities{},
+				Explain:      schema.LeafCapability{},
+			},
+			Mutation: schema.MutationCapabilities{
+				Explain: schema.LeafCapability{},
 			},
-			Mutation: schema.MutationCapabilities{},
 		},
 	}
 	rawCapabilities, err := json.Marshal(restCapabilities)
@@ -53,23 +55,32 @@ func (c *RESTConnector) ParseConfiguration(ctx context.Context, configurationDir
 	}
 	c.capabilities = schema.NewRawCapabilitiesResponseUnsafe(rawCapabilities)
 
-	config, err := parseConfiguration(configurationDir)
+	config, err := configuration.ReadConfigurationFile(configurationDir)
 	if err != nil {
 		return nil, err
 	}
 
 	logger := connector.GetLogger(ctx)
-	schemas, errs := BuildSchemaFiles(configurationDir, config.Files, logger)
-	if len(errs) > 0 {
-		printSchemaValidationError(logger, errs)
-		return nil, errBuildSchemaFailed
+	schemas, err := configuration.ReadSchemaOutputFile(configurationDir, config.Output, logger)
+	if err != nil {
+		return nil, err
 	}
 
-	if errs := c.ApplyNDCRestSchemas(schemas); len(errs) > 0 {
-		printSchemaValidationError(logger, errs)
+	var errs map[string][]string
+	if schemas == nil {
+		schemas, errs = configuration.BuildSchemaFromConfig(config, configurationDir, logger)
+		if len(errs) > 0 {
+			printSchemaValidationError(logger, errs)
+			return nil, errBuildSchemaFailed
+		}
+	}
+
+	if err := c.ApplyNDCRestSchemas(config, schemas, logger); err != nil {
 		return nil, errInvalidSchema
 	}
 
+	c.config = config
+
 	return config, nil
 }
 
@@ -82,7 +93,10 @@ func (c *RESTConnector) ParseConfiguration(ctx context.Context, configurationDir
 // connector-specific metrics with the metrics registry.
 func (c *RESTConnector) TryInitState(ctx context.Context, configuration *configuration.Configuration, metrics *connector.TelemetryState) (*State, error) {
 	c.client.SetTracer(metrics.Tracer)
-	return &State{}, nil
+
+	return &State{
+		Tracer: metrics.Tracer,
+	}, nil
 }
 
 // HealthCheck checks the health of the connector.
@@ -99,41 +113,3 @@ func (c *RESTConnector) HealthCheck(ctx context.Context, configuration *configur
 func (c *RESTConnector) GetCapabilities(configuration *configuration.Configuration) schema.CapabilitiesResponseMarshaler {
 	return c.capabilities
 }
-
-func parseConfiguration(configurationDir string) (*configuration.Configuration, error) {
-	var config configuration.Configuration
-	jsonBytes, err := os.ReadFile(configurationDir + "/config.json")
-	if err == nil {
-		if err = json.Unmarshal(jsonBytes, &config); err != nil {
-			return nil, err
-		}
-		return &config, nil
-	}
-
-	if !os.IsNotExist(err) {
-		return nil, err
-	}
-
-	// try to read and parse yaml file
-	yamlBytes, err := os.ReadFile(configurationDir + "/config.yaml")
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return nil, err
-		}
-		yamlBytes, err = os.ReadFile(configurationDir + "/config.yml")
-	}
-
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, fmt.Errorf("the config.{json,yaml,yml} file does not exist at %s", configurationDir)
-		} else {
-			return nil, err
-		}
-	}
-
-	if err = yaml.Unmarshal(yamlBytes, &config); err != nil {
-		return nil, err
-	}
-
-	return &config, nil
-}