Skip to content

Commit

Permalink
feat: Output JSON when applying an overlay to a JSON doc (#950)
Browse files Browse the repository at this point in the history
Improved JSON schema support in the studio:
- `run` will now respect input filetype when applying overlays.
Specifically, input `json` specs when overlaid will write `json` out.
Note this is only relevant / visible to the user now because of the
studio
- `run` will now reformat single-line input specs. This helps with both
the studio and linting output
- Adds an integration test confirming that we can pull json documents
from the registry

Related speakeasy-api/sdk-gen-config#58
  • Loading branch information
chase-crumbaugh committed Sep 19, 2024
1 parent 8a61986 commit e2b2fa9
Show file tree
Hide file tree
Showing 9 changed files with 192 additions and 63 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/speakeasy-api/huh v1.1.1
github.com/speakeasy-api/openapi-generation/v2 v2.420.2
github.com/speakeasy-api/openapi-overlay v0.9.0
github.com/speakeasy-api/sdk-gen-config v1.23.0
github.com/speakeasy-api/sdk-gen-config v1.23.1
github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.1
github.com/speakeasy-api/speakeasy-core v0.15.4
github.com/speakeasy-api/speakeasy-proxy v0.0.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -524,8 +524,8 @@ github.com/speakeasy-api/openapi-generation/v2 v2.420.2 h1:/5xezA7qwaKpM1aDM4o4V
github.com/speakeasy-api/openapi-generation/v2 v2.420.2/go.mod h1:AmiD1HPK7QC46gq3/kKvgtdg22qr1BJO+HuV4vZT2RU=
github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg=
github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc=
github.com/speakeasy-api/sdk-gen-config v1.23.0 h1:glWD27mclxMFL7wSspIPJjkie9K96v0eZ/Tfmc9iuQw=
github.com/speakeasy-api/sdk-gen-config v1.23.0/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
github.com/speakeasy-api/sdk-gen-config v1.23.1 h1:aWdBKehsi9xUMEVbU5UmZ9OBaHxEnPKUnFNMQpFLSFM=
github.com/speakeasy-api/sdk-gen-config v1.23.1/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.1 h1:BvRDFYa/Ibpcs8r/2wdS8xTrOBi6sWyI+p3LRasU0lA=
github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.1/go.mod h1:b4fiZ1Wid0JHwwiYqhaPifDwjmC15uiN7A8Cmid+9kw=
github.com/speakeasy-api/speakeasy-core v0.15.4 h1:/l++jE9Bx6TmmJwdDYwiWcqQOuvIarhng4/c67AQ92s=
Expand Down
50 changes: 50 additions & 0 deletions integration/workflow_registry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package integration_tests
import (
"crypto/md5"
"fmt"
"gopkg.in/yaml.v3"
"os"
"path/filepath"
"testing"
Expand Down Expand Up @@ -118,6 +119,55 @@ func TestRegistryFlow(t *testing.T) {
require.NoError(t, cmdErr)
}

func TestRegistryFlow_JSON(t *testing.T) {
t.Parallel()
temp := setupTestDir(t)

// Create a basic workflow file
workflowFile := &workflow.Workflow{
Version: workflow.WorkflowVersion,
Sources: map[string]workflow.Source{
"test-source": {
Inputs: []workflow.Document{
{Location: "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v3.0/petstore.json"},
},
},
},
Targets: map[string]workflow.Target{
"test-target": {
Target: "typescript",
Source: "test-source",
},
},
}

err := os.MkdirAll(filepath.Join(temp, ".speakeasy"), 0o755)
require.NoError(t, err)
b, err := yaml.Marshal(workflowFile)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(temp, ".speakeasy", "workflow.yaml"), b, 0o644))

// Run the initial generation
initialArgs := []string{"run", "-t", "all", "--force", "--pinned", "--skip-versioning", "--skip-compile"}
cmdErr := execute(t, temp, initialArgs...).Run()
require.NoError(t, cmdErr)

// Get the registry location and set it to the source input
workflowFile, _, err = workflow.Load(temp)
require.NoError(t, err)
registryLocation := workflowFile.Sources["test-source"].Registry.Location.String()
require.True(t, len(registryLocation) > 0, "registry location should be set")

print(registryLocation)

workflowFile.Sources["test-source"].Inputs[0].Location = workflow.LocationString(registryLocation)
require.NoError(t, workflow.Save(temp, workflowFile))

// Re-run the generation. It should work.
cmdErr = executeI(t, temp, initialArgs...).Run()
require.NoError(t, cmdErr)
}

func calculateChecksums(dir string) (map[string]string, error) {
checksums := make(map[string]string)
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
Expand Down
44 changes: 2 additions & 42 deletions internal/overlay/overlay.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
package overlay

import (
"bytes"
"context"
"fmt"
"github.com/pb33f/libopenapi/json"
"github.com/speakeasy-api/openapi-overlay/pkg/loader"
"github.com/speakeasy-api/openapi-overlay/pkg/overlay"
"github.com/speakeasy-api/speakeasy-core/openapi"
"github.com/speakeasy-api/speakeasy/internal/log"
"github.com/speakeasy-api/speakeasy/internal/utils"
"gopkg.in/yaml.v3"
"github.com/speakeasy-api/speakeasy/internal/schemas"
"io"
)

Expand Down Expand Up @@ -81,7 +77,7 @@ func Apply(schema string, overlayFile string, yamlOut bool, w io.Writer, strict
}
}

bytes, err := Render(ys, schema, yamlOut)
bytes, err := schemas.Render(ys, schema, yamlOut)
if err != nil {
return fmt.Errorf("failed to Render document: %w", err)
}
Expand All @@ -92,39 +88,3 @@ func Apply(schema string, overlayFile string, yamlOut bool, w io.Writer, strict

return nil
}

func Render(y *yaml.Node, schemaPath string, yamlOut bool) ([]byte, error) {
yamlIn := utils.HasYAMLExt(schemaPath)

if yamlIn && yamlOut {
var res bytes.Buffer
encoder := yaml.NewEncoder(&res)
// Note: would love to make this generic but the indentation information isn't in go-yaml nodes
// https://github.com/go-yaml/yaml/issues/899
encoder.SetIndent(2)
if err := encoder.Encode(y); err != nil {
return nil, fmt.Errorf("failed to encode YAML: %w", err)
}
return res.Bytes(), nil
}

// Preserves key ordering
specBytes, err := json.YAMLNodeToJSON(y, " ")
if err != nil {
return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err)
}

if yamlOut {
// Use libopenapi to convert JSON to YAML to preserve key ordering
_, model, err := openapi.Load(specBytes, schemaPath)

yamlBytes, err := model.Model.Render()
if err != nil {
return nil, fmt.Errorf("failed to Render YAML: %w", err)
}

return yamlBytes, nil
} else {
return specBytes, nil
}
}
72 changes: 64 additions & 8 deletions internal/run/source.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package run

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/fs"
Expand Down Expand Up @@ -34,7 +36,7 @@ import (
"github.com/speakeasy-api/speakeasy/internal/log"
"github.com/speakeasy-api/speakeasy/internal/overlay"
"github.com/speakeasy-api/speakeasy/internal/reports"
"github.com/speakeasy-api/speakeasy/internal/schema"
"github.com/speakeasy-api/speakeasy/internal/schemas"
"github.com/speakeasy-api/speakeasy/internal/suggest"
"github.com/speakeasy-api/speakeasy/internal/utils"
"github.com/speakeasy-api/speakeasy/internal/validation"
Expand Down Expand Up @@ -147,13 +149,21 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W
if len(source.Overlays) == 0 {
singleLocation = &outputLocation
}
currentDocument, err = schema.ResolveDocument(ctx, source.Inputs[0], singleLocation, rootStep)
currentDocument, err = schemas.ResolveDocument(ctx, source.Inputs[0], singleLocation, rootStep)
if err != nil {
return "", nil, err
}
// In registry bundles specifically we cannot know the exact file output location before pulling the bundle down
if len(source.Overlays) == 0 && source.Inputs[0].IsSpeakeasyRegistry() {
outputLocation = currentDocument
if len(source.Overlays) == 0 {
// In registry bundles specifically we cannot know the exact file output location before pulling the bundle down
if source.Inputs[0].IsSpeakeasyRegistry() {
outputLocation = currentDocument
}
// If we aren't going to touch the document because it's a single input document with no overlay, then check if we should reformat it
// Primarily this is to improve readability of single-line documents in the Studio and Linting output
if reformattedLocation, wasReformatted, err := maybeReformatDocument(ctx, currentDocument, rootStep); err == nil && wasReformatted {
currentDocument = reformattedLocation
outputLocation = reformattedLocation
}
}
} else {
mergeStep := rootStep.NewSubstep("Merge Documents")
Expand All @@ -167,7 +177,7 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W

inSchemas := []string{}
for _, input := range source.Inputs {
resolvedPath, err := schema.ResolveDocument(ctx, input, nil, mergeStep)
resolvedPath, err := schemas.ResolveDocument(ctx, input, nil, mergeStep)
if err != nil {
return "", nil, err
}
Expand Down Expand Up @@ -199,7 +209,7 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W
for _, overlay := range source.Overlays {
overlayFilePath := ""
if overlay.Document != nil {
overlayFilePath, err = schema.ResolveDocument(ctx, *overlay.Document, nil, overlayStep)
overlayFilePath, err = schemas.ResolveDocument(ctx, *overlay.Document, nil, overlayStep)
if err != nil {
return "", nil, err
}
Expand All @@ -220,8 +230,8 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W
return "", nil, err
}
}
overlaySchemas = append(overlaySchemas, overlayFilePath)

overlaySchemas = append(overlaySchemas, overlayFilePath)
}

overlayStep.NewSubstep(fmt.Sprintf("Apply %d overlay(s)", len(source.Overlays)))
Expand Down Expand Up @@ -777,3 +787,49 @@ var randStringBytes = func(n int) string {
func getTempApplyPath(path string) string {
return filepath.Join(workflow.GetTempDir(), fmt.Sprintf("applied_%s%s", randStringBytes(10), filepath.Ext(path)))
}

func maybeReformatDocument(ctx context.Context, documentPath string, rootStep *workflowTracking.WorkflowStep) (string, bool, error) {
content, err := os.ReadFile(documentPath)
if err != nil {
log.From(ctx).Warnf("Failed to read document: %v", err)
return documentPath, false, err
}

// Check if the file is only a single line
if bytes.Count(content, []byte("\n")) == 0 {
reformatStep := rootStep.NewSubstep("Reformatting Single-Line Document")

returnErr := func(err error) (string, bool, error) {
log.From(ctx).Warnf("Failed to reformat document: %v", err)
reformatStep.Fail()
return documentPath, false, err
}

isJSON := json.Valid(content)

reformattedContent, err := schemas.Format(ctx, documentPath, !isJSON)
if err != nil {
return returnErr(fmt.Errorf("failed to format document: %w", err))
}

// Write reformatted content to a new temporary file
if err := os.MkdirAll(workflow.GetTempDir(), os.ModePerm); err != nil {
return returnErr(fmt.Errorf("failed to create temp dir: %w", err))
}
tempFile, err := os.CreateTemp(workflow.GetTempDir(), "reformatted*"+filepath.Ext(documentPath))
if err != nil {
return returnErr(fmt.Errorf("failed to create temporary file: %w", err))
}
defer tempFile.Close()

if _, err := tempFile.Write(reformattedContent); err != nil {
return returnErr(fmt.Errorf("failed to write reformatted content: %w", err))
}

reformatStep.Succeed()
log.From(ctx).Infof("Document reformatted and saved to: %s", tempFile.Name())
return tempFile.Name(), true, nil
}

return documentPath, false, nil
}
14 changes: 10 additions & 4 deletions internal/schema/document.go → internal/schemas/document.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package schema
package schemas

import (
"context"
Expand All @@ -8,6 +8,7 @@ import (
"github.com/speakeasy-api/sdk-gen-config/workflow"
"github.com/speakeasy-api/speakeasy-core/openapi"
"github.com/speakeasy-api/speakeasy/internal/download"
"github.com/speakeasy-api/speakeasy/internal/utils"
"github.com/speakeasy-api/speakeasy/internal/workflowTracking"
"github.com/speakeasy-api/speakeasy/registry"
)
Expand All @@ -29,14 +30,19 @@ func ResolveDocument(ctx context.Context, d workflow.Document, outputLocation *s
}

location := workflow.GetTempDir()
if outputLocation != nil {
location = *outputLocation
}
documentOut, err := registry.ResolveSpeakeasyRegistryBundle(ctx, d, location)
if err != nil {
return "", err
}

// Note that workflows with inputs from the registry will not work with $refs to other files in the bundle
if outputLocation != nil {
// Copy actual document out of bundle over to outputLocation
if err := utils.CopyFile(documentOut.LocalFilePath, *outputLocation); err != nil {
return "", err
}
}

return documentOut.LocalFilePath, nil
} else if d.IsRemote() {
step.NewSubstep("Downloading remote document")
Expand Down
58 changes: 58 additions & 0 deletions internal/schemas/format.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package schemas

import (
"bytes"
"context"
"fmt"
"github.com/pb33f/libopenapi/json"
"github.com/speakeasy-api/speakeasy-core/openapi"
"github.com/speakeasy-api/speakeasy/internal/utils"
"gopkg.in/yaml.v3"
)

// Format reformats a document to the desired output format while preserving key ordering
// Can be used to convert output types, or improve readability (e.g. prettifying single-line documents)
func Format(ctx context.Context, schemaPath string, yamlOut bool) ([]byte, error) {
_, _, model, err := LoadDocument(ctx, schemaPath)
if err != nil {
return nil, fmt.Errorf("failed to parse document: %w", err)
}

return Render(model.Index.GetRootNode(), schemaPath, yamlOut)
}

func Render(y *yaml.Node, schemaPath string, yamlOut bool) ([]byte, error) {
yamlIn := utils.HasYAMLExt(schemaPath)

if yamlIn && yamlOut {
var res bytes.Buffer
encoder := yaml.NewEncoder(&res)
// Note: would love to make this generic but the indentation information isn't in go-yaml nodes
// https://github.com/go-yaml/yaml/issues/899
encoder.SetIndent(2)
if err := encoder.Encode(y); err != nil {
return nil, fmt.Errorf("failed to encode YAML: %w", err)
}
return res.Bytes(), nil
}

// Preserves key ordering
specBytes, err := json.YAMLNodeToJSON(y, " ")
if err != nil {
return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err)
}

if yamlOut {
// Use libopenapi to convert JSON to YAML to preserve key ordering
_, model, err := openapi.Load(specBytes, schemaPath)

yamlBytes, err := model.Model.Render()
if err != nil {
return nil, fmt.Errorf("failed to Render YAML: %w", err)
}

return yamlBytes, nil
} else {
return specBytes, nil
}
}
4 changes: 2 additions & 2 deletions internal/suggest/diagnose.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ import (
"context"
"github.com/speakeasy-api/speakeasy-core/openapi"
"github.com/speakeasy-api/speakeasy-core/suggestions"
"github.com/speakeasy-api/speakeasy/internal/schema"
"github.com/speakeasy-api/speakeasy/internal/schemas"
)

func Diagnose(ctx context.Context, schemaPath string) (suggestions.Diagnosis, error) {
data, _, _, err := schema.LoadDocument(ctx, schemaPath)
data, _, _, err := schemas.LoadDocument(ctx, schemaPath)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit e2b2fa9

Please sign in to comment.