From e2b2fa961cd2e1e9b97281b61e7cde0a3d32d35a Mon Sep 17 00:00:00 2001 From: Chase Date: Thu, 19 Sep 2024 15:49:21 -0700 Subject: [PATCH] feat: Output JSON when applying an overlay to a JSON doc (#950) Improved JSON schema support in the studio: - `run` will now respect input filetype when applying overlays. Specifically, input `json` specs when overlaid will write `json` out. Note this is only relevant / visible to the user now because of the studio - `run` will now reformat single-line input specs. This helps with both the studio and linting output - Adds an integration test confirming that we can pull json documents from the registry Related https://github.com/speakeasy-api/sdk-gen-config/pull/58 --- go.mod | 2 +- go.sum | 4 +- integration/workflow_registry_test.go | 50 ++++++++++++++++ internal/overlay/overlay.go | 44 +-------------- internal/run/source.go | 72 +++++++++++++++++++++--- internal/{schema => schemas}/document.go | 14 +++-- internal/schemas/format.go | 58 +++++++++++++++++++ internal/suggest/diagnose.go | 4 +- internal/suggest/suggest.go | 7 +-- 9 files changed, 192 insertions(+), 63 deletions(-) rename internal/{schema => schemas}/document.go (82%) create mode 100644 internal/schemas/format.go diff --git a/go.mod b/go.mod index 71ada8f1..55c875dd 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/speakeasy-api/huh v1.1.1 github.com/speakeasy-api/openapi-generation/v2 v2.420.2 github.com/speakeasy-api/openapi-overlay v0.9.0 - github.com/speakeasy-api/sdk-gen-config v1.23.0 + github.com/speakeasy-api/sdk-gen-config v1.23.1 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.1 github.com/speakeasy-api/speakeasy-core v0.15.4 github.com/speakeasy-api/speakeasy-proxy v0.0.2 diff --git a/go.sum b/go.sum index 08cc2d6a..938c4cc7 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/speakeasy-api/openapi-generation/v2 v2.420.2 h1:/5xezA7qwaKpM1aDM4o4V github.com/speakeasy-api/openapi-generation/v2 v2.420.2/go.mod h1:AmiD1HPK7QC46gq3/kKvgtdg22qr1BJO+HuV4vZT2RU= github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg= github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc= -github.com/speakeasy-api/sdk-gen-config v1.23.0 h1:glWD27mclxMFL7wSspIPJjkie9K96v0eZ/Tfmc9iuQw= -github.com/speakeasy-api/sdk-gen-config v1.23.0/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ= +github.com/speakeasy-api/sdk-gen-config v1.23.1 h1:aWdBKehsi9xUMEVbU5UmZ9OBaHxEnPKUnFNMQpFLSFM= +github.com/speakeasy-api/sdk-gen-config v1.23.1/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ= github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.1 h1:BvRDFYa/Ibpcs8r/2wdS8xTrOBi6sWyI+p3LRasU0lA= github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.1/go.mod h1:b4fiZ1Wid0JHwwiYqhaPifDwjmC15uiN7A8Cmid+9kw= github.com/speakeasy-api/speakeasy-core v0.15.4 h1:/l++jE9Bx6TmmJwdDYwiWcqQOuvIarhng4/c67AQ92s= diff --git a/integration/workflow_registry_test.go b/integration/workflow_registry_test.go index 5e5dd066..42ab87fc 100644 --- a/integration/workflow_registry_test.go +++ b/integration/workflow_registry_test.go @@ -3,6 +3,7 @@ package integration_tests import ( "crypto/md5" "fmt" + "gopkg.in/yaml.v3" "os" "path/filepath" "testing" @@ -118,6 +119,55 @@ func TestRegistryFlow(t *testing.T) { require.NoError(t, cmdErr) } +func TestRegistryFlow_JSON(t *testing.T) { + t.Parallel() + temp := setupTestDir(t) + + // Create a basic workflow file + workflowFile := &workflow.Workflow{ + Version: workflow.WorkflowVersion, + Sources: map[string]workflow.Source{ + "test-source": { + Inputs: []workflow.Document{ + {Location: "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v3.0/petstore.json"}, + }, + }, + }, + Targets: map[string]workflow.Target{ + "test-target": { + Target: "typescript", + Source: "test-source", + }, + }, + } + + err := os.MkdirAll(filepath.Join(temp, ".speakeasy"), 0o755) + require.NoError(t, err) + b, err := yaml.Marshal(workflowFile) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(temp, ".speakeasy", "workflow.yaml"), b, 0o644)) + + // Run the initial generation + initialArgs := []string{"run", "-t", "all", "--force", "--pinned", "--skip-versioning", "--skip-compile"} + cmdErr := execute(t, temp, initialArgs...).Run() + require.NoError(t, cmdErr) + + // Get the registry location and set it to the source input + workflowFile, _, err = workflow.Load(temp) + require.NoError(t, err) + registryLocation := workflowFile.Sources["test-source"].Registry.Location.String() + require.True(t, len(registryLocation) > 0, "registry location should be set") + + print(registryLocation) + + workflowFile.Sources["test-source"].Inputs[0].Location = workflow.LocationString(registryLocation) + require.NoError(t, workflow.Save(temp, workflowFile)) + + // Re-run the generation. It should work. + cmdErr = executeI(t, temp, initialArgs...).Run() + require.NoError(t, cmdErr) +} + func calculateChecksums(dir string) (map[string]string, error) { checksums := make(map[string]string) err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { diff --git a/internal/overlay/overlay.go b/internal/overlay/overlay.go index a60c7d7e..1e83ae0d 100644 --- a/internal/overlay/overlay.go +++ b/internal/overlay/overlay.go @@ -1,16 +1,12 @@ package overlay import ( - "bytes" "context" "fmt" - "github.com/pb33f/libopenapi/json" "github.com/speakeasy-api/openapi-overlay/pkg/loader" "github.com/speakeasy-api/openapi-overlay/pkg/overlay" - "github.com/speakeasy-api/speakeasy-core/openapi" "github.com/speakeasy-api/speakeasy/internal/log" - "github.com/speakeasy-api/speakeasy/internal/utils" - "gopkg.in/yaml.v3" + "github.com/speakeasy-api/speakeasy/internal/schemas" "io" ) @@ -81,7 +77,7 @@ func Apply(schema string, overlayFile string, yamlOut bool, w io.Writer, strict } } - bytes, err := Render(ys, schema, yamlOut) + bytes, err := schemas.Render(ys, schema, yamlOut) if err != nil { return fmt.Errorf("failed to Render document: %w", err) } @@ -92,39 +88,3 @@ func Apply(schema string, overlayFile string, yamlOut bool, w io.Writer, strict return nil } - -func Render(y *yaml.Node, schemaPath string, yamlOut bool) ([]byte, error) { - yamlIn := utils.HasYAMLExt(schemaPath) - - if yamlIn && yamlOut { - var res bytes.Buffer - encoder := yaml.NewEncoder(&res) - // Note: would love to make this generic but the indentation information isn't in go-yaml nodes - // https://github.com/go-yaml/yaml/issues/899 - encoder.SetIndent(2) - if err := encoder.Encode(y); err != nil { - return nil, fmt.Errorf("failed to encode YAML: %w", err) - } - return res.Bytes(), nil - } - - // Preserves key ordering - specBytes, err := json.YAMLNodeToJSON(y, " ") - if err != nil { - return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err) - } - - if yamlOut { - // Use libopenapi to convert JSON to YAML to preserve key ordering - _, model, err := openapi.Load(specBytes, schemaPath) - - yamlBytes, err := model.Model.Render() - if err != nil { - return nil, fmt.Errorf("failed to Render YAML: %w", err) - } - - return yamlBytes, nil - } else { - return specBytes, nil - } -} diff --git a/internal/run/source.go b/internal/run/source.go index a561c4ee..31c7cf19 100644 --- a/internal/run/source.go +++ b/internal/run/source.go @@ -1,7 +1,9 @@ package run import ( + "bytes" "context" + "encoding/json" "fmt" "io" "io/fs" @@ -34,7 +36,7 @@ import ( "github.com/speakeasy-api/speakeasy/internal/log" "github.com/speakeasy-api/speakeasy/internal/overlay" "github.com/speakeasy-api/speakeasy/internal/reports" - "github.com/speakeasy-api/speakeasy/internal/schema" + "github.com/speakeasy-api/speakeasy/internal/schemas" "github.com/speakeasy-api/speakeasy/internal/suggest" "github.com/speakeasy-api/speakeasy/internal/utils" "github.com/speakeasy-api/speakeasy/internal/validation" @@ -147,13 +149,21 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W if len(source.Overlays) == 0 { singleLocation = &outputLocation } - currentDocument, err = schema.ResolveDocument(ctx, source.Inputs[0], singleLocation, rootStep) + currentDocument, err = schemas.ResolveDocument(ctx, source.Inputs[0], singleLocation, rootStep) if err != nil { return "", nil, err } - // In registry bundles specifically we cannot know the exact file output location before pulling the bundle down - if len(source.Overlays) == 0 && source.Inputs[0].IsSpeakeasyRegistry() { - outputLocation = currentDocument + if len(source.Overlays) == 0 { + // In registry bundles specifically we cannot know the exact file output location before pulling the bundle down + if source.Inputs[0].IsSpeakeasyRegistry() { + outputLocation = currentDocument + } + // If we aren't going to touch the document because it's a single input document with no overlay, then check if we should reformat it + // Primarily this is to improve readability of single-line documents in the Studio and Linting output + if reformattedLocation, wasReformatted, err := maybeReformatDocument(ctx, currentDocument, rootStep); err == nil && wasReformatted { + currentDocument = reformattedLocation + outputLocation = reformattedLocation + } } } else { mergeStep := rootStep.NewSubstep("Merge Documents") @@ -167,7 +177,7 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W inSchemas := []string{} for _, input := range source.Inputs { - resolvedPath, err := schema.ResolveDocument(ctx, input, nil, mergeStep) + resolvedPath, err := schemas.ResolveDocument(ctx, input, nil, mergeStep) if err != nil { return "", nil, err } @@ -199,7 +209,7 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W for _, overlay := range source.Overlays { overlayFilePath := "" if overlay.Document != nil { - overlayFilePath, err = schema.ResolveDocument(ctx, *overlay.Document, nil, overlayStep) + overlayFilePath, err = schemas.ResolveDocument(ctx, *overlay.Document, nil, overlayStep) if err != nil { return "", nil, err } @@ -220,8 +230,8 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W return "", nil, err } } - overlaySchemas = append(overlaySchemas, overlayFilePath) + overlaySchemas = append(overlaySchemas, overlayFilePath) } overlayStep.NewSubstep(fmt.Sprintf("Apply %d overlay(s)", len(source.Overlays))) @@ -777,3 +787,49 @@ var randStringBytes = func(n int) string { func getTempApplyPath(path string) string { return filepath.Join(workflow.GetTempDir(), fmt.Sprintf("applied_%s%s", randStringBytes(10), filepath.Ext(path))) } + +func maybeReformatDocument(ctx context.Context, documentPath string, rootStep *workflowTracking.WorkflowStep) (string, bool, error) { + content, err := os.ReadFile(documentPath) + if err != nil { + log.From(ctx).Warnf("Failed to read document: %v", err) + return documentPath, false, err + } + + // Check if the file is only a single line + if bytes.Count(content, []byte("\n")) == 0 { + reformatStep := rootStep.NewSubstep("Reformatting Single-Line Document") + + returnErr := func(err error) (string, bool, error) { + log.From(ctx).Warnf("Failed to reformat document: %v", err) + reformatStep.Fail() + return documentPath, false, err + } + + isJSON := json.Valid(content) + + reformattedContent, err := schemas.Format(ctx, documentPath, !isJSON) + if err != nil { + return returnErr(fmt.Errorf("failed to format document: %w", err)) + } + + // Write reformatted content to a new temporary file + if err := os.MkdirAll(workflow.GetTempDir(), os.ModePerm); err != nil { + return returnErr(fmt.Errorf("failed to create temp dir: %w", err)) + } + tempFile, err := os.CreateTemp(workflow.GetTempDir(), "reformatted*"+filepath.Ext(documentPath)) + if err != nil { + return returnErr(fmt.Errorf("failed to create temporary file: %w", err)) + } + defer tempFile.Close() + + if _, err := tempFile.Write(reformattedContent); err != nil { + return returnErr(fmt.Errorf("failed to write reformatted content: %w", err)) + } + + reformatStep.Succeed() + log.From(ctx).Infof("Document reformatted and saved to: %s", tempFile.Name()) + return tempFile.Name(), true, nil + } + + return documentPath, false, nil +} diff --git a/internal/schema/document.go b/internal/schemas/document.go similarity index 82% rename from internal/schema/document.go rename to internal/schemas/document.go index d0cc3da7..eec1abea 100644 --- a/internal/schema/document.go +++ b/internal/schemas/document.go @@ -1,4 +1,4 @@ -package schema +package schemas import ( "context" @@ -8,6 +8,7 @@ import ( "github.com/speakeasy-api/sdk-gen-config/workflow" "github.com/speakeasy-api/speakeasy-core/openapi" "github.com/speakeasy-api/speakeasy/internal/download" + "github.com/speakeasy-api/speakeasy/internal/utils" "github.com/speakeasy-api/speakeasy/internal/workflowTracking" "github.com/speakeasy-api/speakeasy/registry" ) @@ -29,14 +30,19 @@ func ResolveDocument(ctx context.Context, d workflow.Document, outputLocation *s } location := workflow.GetTempDir() - if outputLocation != nil { - location = *outputLocation - } documentOut, err := registry.ResolveSpeakeasyRegistryBundle(ctx, d, location) if err != nil { return "", err } + // Note that workflows with inputs from the registry will not work with $refs to other files in the bundle + if outputLocation != nil { + // Copy actual document out of bundle over to outputLocation + if err := utils.CopyFile(documentOut.LocalFilePath, *outputLocation); err != nil { + return "", err + } + } + return documentOut.LocalFilePath, nil } else if d.IsRemote() { step.NewSubstep("Downloading remote document") diff --git a/internal/schemas/format.go b/internal/schemas/format.go new file mode 100644 index 00000000..cf6b8153 --- /dev/null +++ b/internal/schemas/format.go @@ -0,0 +1,58 @@ +package schemas + +import ( + "bytes" + "context" + "fmt" + "github.com/pb33f/libopenapi/json" + "github.com/speakeasy-api/speakeasy-core/openapi" + "github.com/speakeasy-api/speakeasy/internal/utils" + "gopkg.in/yaml.v3" +) + +// Format reformats a document to the desired output format while preserving key ordering +// Can be used to convert output types, or improve readability (e.g. prettifying single-line documents) +func Format(ctx context.Context, schemaPath string, yamlOut bool) ([]byte, error) { + _, _, model, err := LoadDocument(ctx, schemaPath) + if err != nil { + return nil, fmt.Errorf("failed to parse document: %w", err) + } + + return Render(model.Index.GetRootNode(), schemaPath, yamlOut) +} + +func Render(y *yaml.Node, schemaPath string, yamlOut bool) ([]byte, error) { + yamlIn := utils.HasYAMLExt(schemaPath) + + if yamlIn && yamlOut { + var res bytes.Buffer + encoder := yaml.NewEncoder(&res) + // Note: would love to make this generic but the indentation information isn't in go-yaml nodes + // https://github.com/go-yaml/yaml/issues/899 + encoder.SetIndent(2) + if err := encoder.Encode(y); err != nil { + return nil, fmt.Errorf("failed to encode YAML: %w", err) + } + return res.Bytes(), nil + } + + // Preserves key ordering + specBytes, err := json.YAMLNodeToJSON(y, " ") + if err != nil { + return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err) + } + + if yamlOut { + // Use libopenapi to convert JSON to YAML to preserve key ordering + _, model, err := openapi.Load(specBytes, schemaPath) + + yamlBytes, err := model.Model.Render() + if err != nil { + return nil, fmt.Errorf("failed to Render YAML: %w", err) + } + + return yamlBytes, nil + } else { + return specBytes, nil + } +} diff --git a/internal/suggest/diagnose.go b/internal/suggest/diagnose.go index f5e5affd..8d8d5aa0 100644 --- a/internal/suggest/diagnose.go +++ b/internal/suggest/diagnose.go @@ -4,11 +4,11 @@ import ( "context" "github.com/speakeasy-api/speakeasy-core/openapi" "github.com/speakeasy-api/speakeasy-core/suggestions" - "github.com/speakeasy-api/speakeasy/internal/schema" + "github.com/speakeasy-api/speakeasy/internal/schemas" ) func Diagnose(ctx context.Context, schemaPath string) (suggestions.Diagnosis, error) { - data, _, _, err := schema.LoadDocument(ctx, schemaPath) + data, _, _, err := schemas.LoadDocument(ctx, schemaPath) if err != nil { return nil, err } diff --git a/internal/suggest/suggest.go b/internal/suggest/suggest.go index a0ea42fc..c9018a38 100644 --- a/internal/suggest/suggest.go +++ b/internal/suggest/suggest.go @@ -19,8 +19,7 @@ import ( "github.com/speakeasy-api/openapi-overlay/pkg/overlay" "github.com/speakeasy-api/speakeasy-client-sdk-go/v3/pkg/models/shared" "github.com/speakeasy-api/speakeasy/internal/log" - overlayUtil "github.com/speakeasy-api/speakeasy/internal/overlay" - "github.com/speakeasy-api/speakeasy/internal/schema" + "github.com/speakeasy-api/speakeasy/internal/schemas" speakeasy "github.com/speakeasy-api/speakeasy-client-sdk-go/v3" "github.com/speakeasy-api/speakeasy-client-sdk-go/v3/pkg/models/operations" @@ -34,7 +33,7 @@ func SuggestOperationIDsAndWrite(ctx context.Context, schemaLocation string, asO yamlOut = true } - schemaBytes, _, model, err := schema.LoadDocument(ctx, schemaLocation) + schemaBytes, _, model, err := schemas.LoadDocument(ctx, schemaLocation) if err != nil { return err } @@ -64,7 +63,7 @@ func SuggestOperationIDsAndWrite(ctx context.Context, schemaLocation string, asO return err } - finalBytes, err := overlayUtil.Render(root, schemaLocation, yamlOut) + finalBytes, err := schemas.Render(root, schemaLocation, yamlOut) if err != nil { return err }