chore: support environment variables in workflow files (#939)

cmd/openapi.go

@@ -263,7 +263,7 @@ func processRegistryBundles(ctx context.Context, flags OpenAPIDiffFlags) (bool,
 	hasRegistrySchema := false
 	if strings.Contains(oldSchema, "registry.speakeasyapi.dev/") {
 		document := workflow.Document{
-			Location: oldSchema,
+			Location: workflow.LocationString(oldSchema),
 		}
 
 		output := workflow.GetTempDir()
@@ -285,7 +285,7 @@ func processRegistryBundles(ctx context.Context, flags OpenAPIDiffFlags) (bool,
 
 	if strings.Contains(newSchema, "registry.speakeasyapi.dev/") {
 		document := workflow.Document{
-			Location: newSchema,
+			Location: workflow.LocationString(newSchema),
 		}
 
 		output := workflow.GetTempDir()

cmd/quickstart.go

@@ -203,7 +203,7 @@ func quickstartExec(ctx context.Context, flags QuickstartFlags) error {
 	var sourceName string
 	for name, source := range quickstartObj.WorkflowFile.Sources {
 		sourceName = name
-		resolvedSchema = source.Inputs[0].Location
+		resolvedSchema = source.Inputs[0].Location.Resolve()
 	}
 
 	// If we are referencing a local schema, set a relative path for the new out directory
@@ -217,7 +217,7 @@ func quickstartExec(ctx context.Context, flags QuickstartFlags) error {
 		if err != nil {
 			return err
 		}
-		quickstartObj.WorkflowFile.Sources[sourceName].Inputs[0].Location = referencePath
+		quickstartObj.WorkflowFile.Sources[sourceName].Inputs[0].Location = workflow.LocationString(referencePath)
 	}
 
 	if quickstartObj.IsUsingSampleOpenAPISpec {
@@ -232,7 +232,7 @@ func quickstartExec(ctx context.Context, flags QuickstartFlags) error {
 		if err != nil {
 			return err
 		}
-		quickstartObj.WorkflowFile.Sources[sourceName].Inputs[0].Location = referencePath
+		quickstartObj.WorkflowFile.Sources[sourceName].Inputs[0].Location = workflow.LocationString(referencePath)
 	}
 
 	if err := workflow.Save(outDir, quickstartObj.WorkflowFile); err != nil {

go.mod

@@ -26,9 +26,9 @@ require (
 	github.com/speakeasy-api/huh v1.1.1
 	github.com/speakeasy-api/openapi-generation/v2 v2.415.6
 	github.com/speakeasy-api/openapi-overlay v0.9.0
-	github.com/speakeasy-api/sdk-gen-config v1.21.1
+	github.com/speakeasy-api/sdk-gen-config v1.22.0
 	github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.0
-	github.com/speakeasy-api/speakeasy-core v0.15.2
+	github.com/speakeasy-api/speakeasy-core v0.15.3
 	github.com/speakeasy-api/speakeasy-proxy v0.0.2
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -520,20 +520,16 @@ github.com/speakeasy-api/jsonpath v0.1.1 h1:79gtq+zHYPe9dR1urEwl/PPsvP9nMRTGG1xH
 github.com/speakeasy-api/jsonpath v0.1.1/go.mod h1:Py01TnxRUMhC0/FT954KBiaDY2/kaZv3QMc8JxQaVys=
 github.com/speakeasy-api/libopenapi v0.0.0-20240814113924-cc96d2bc2826 h1:IT86QeGi61zmS/iKkf5rSzxYMO1TDLN/to4ZWP2DyeI=
 github.com/speakeasy-api/libopenapi v0.0.0-20240814113924-cc96d2bc2826/go.mod h1:EjqZbDvviAL4wH/7DF43JuA17D1YTIydhs8qZmjYrWo=
-github.com/speakeasy-api/openapi-generation/v2 v2.415.0 h1:qCMBFLZOFr8PR86dj9XVucte4dpNh6YYl+oNGz8TWL4=
-github.com/speakeasy-api/openapi-generation/v2 v2.415.0/go.mod h1:AmiD1HPK7QC46gq3/kKvgtdg22qr1BJO+HuV4vZT2RU=
 github.com/speakeasy-api/openapi-generation/v2 v2.415.6 h1:KSZe01bPqpDyyrIoL6e3c+K/TaBHES4LrhSMlk0l8Eo=
 github.com/speakeasy-api/openapi-generation/v2 v2.415.6/go.mod h1:AmiD1HPK7QC46gq3/kKvgtdg22qr1BJO+HuV4vZT2RU=
 github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg=
 github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc=
-github.com/speakeasy-api/sdk-gen-config v1.21.0 h1:YtxJIlSrucm3h6iQRQ2VES7QNIAtNuhdDJQq1ZvOttQ=
-github.com/speakeasy-api/sdk-gen-config v1.21.0/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
-github.com/speakeasy-api/sdk-gen-config v1.21.1 h1:/YqThqWuO+blOcrex+2OO1DKktfHTei5u14VKmJIWwQ=
-github.com/speakeasy-api/sdk-gen-config v1.21.1/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
+github.com/speakeasy-api/sdk-gen-config v1.22.0 h1:BGiTEiseH6xjCbddOTF/SN3CL2rGCsQayf3BxXkTqTA=
+github.com/speakeasy-api/sdk-gen-config v1.22.0/go.mod h1:e9PjnCRHGa4K4EFKVU+kKmihOZjJ2V4utcU+274+bnQ=
 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.0 h1:MLLjjf2x2h9HFZY1Awfl/FO5AqCMPxb+0xPwvbQQ49w=
 github.com/speakeasy-api/speakeasy-client-sdk-go/v3 v3.13.0/go.mod h1:b4fiZ1Wid0JHwwiYqhaPifDwjmC15uiN7A8Cmid+9kw=
-github.com/speakeasy-api/speakeasy-core v0.15.2 h1:Pi2rXydUEdorGbV1x2A3XsOhqdeBD2I50b+Q5I4q8ac=
-github.com/speakeasy-api/speakeasy-core v0.15.2/go.mod h1:m10zRjzdzbTcBBb9e+RweBZ4OIowXCvvvxFB3UEmWYs=
+github.com/speakeasy-api/speakeasy-core v0.15.3 h1:9bBywhqMwBgXUu7UKznLD1ypxTyIKC//01W/Zu3YCfU=
+github.com/speakeasy-api/speakeasy-core v0.15.3/go.mod h1:oKhKOkLTohKir7dt4QSvugF/+9uP9T6ONGv3haF+ne8=
 github.com/speakeasy-api/speakeasy-go-sdk v1.8.1 h1:atzohw12oQ5ipaLb1q7ntTu4vvAgKDJsrvaUoOu6sw0=
 github.com/speakeasy-api/speakeasy-go-sdk v1.8.1/go.mod h1:XbzaM0sMjj8bGooz/uEtNkOh1FQiJK7RFuNG3LPBSAU=
 github.com/speakeasy-api/speakeasy-proxy v0.0.2 h1:u4rQ8lXvuYRCSxiLQGb5JxkZRwNIDlyh+pMFYD6OGjA=

integration/workflow_registry_test.go

@@ -110,7 +110,7 @@ func TestRegistryFlow(t *testing.T) {
 	registryLocation := workflowFile.Sources["test-source"].Registry.Location.String()
 	require.True(t, len(registryLocation) > 0, "registry location should be set")
 
-	workflowFile.Sources["test-source"].Inputs[0].Location = registryLocation
+	workflowFile.Sources["test-source"].Inputs[0].Location = workflow.LocationString(registryLocation)
 	require.NoError(t, workflow.Save(temp, workflowFile))
 
 	// Re-run the generation. It should work.

integration/workflow_test.go

@@ -24,6 +24,41 @@ import (
 // If running locally make sure you are running test functions individually TestGenerationWorkflows, TestSpecWorkflows, etc.
 // If all test groups are run at the same time you will see test failures.
 
+func TestWorkflowWithEnvVar(t *testing.T) {
+	temp := setupTestDir(t)
+
+	// Create workflow file and associated resources
+	workflowFile := &workflow.Workflow{
+		Version: workflow.WorkflowVersion,
+		Sources: make(map[string]workflow.Source),
+		Targets: make(map[string]workflow.Target),
+	}
+	workflowFile.Sources["first-source"] = workflow.Source{
+		Inputs: []workflow.Document{
+			{
+				Location: workflow.LocationString("${MY_ENV_VAR}"),
+			},
+		},
+	}
+	workflowFile.Targets["test-target"] = workflow.Target{
+		Target: "typescript",
+		Source: "first-source",
+	}
+
+	err := os.MkdirAll(filepath.Join(temp, ".speakeasy"), 0o755)
+	require.NoError(t, err)
+	err = workflow.Save(temp, workflowFile)
+
+	require.NoError(t, os.Setenv("MY_ENV_VAR", "spec.yaml"))
+	require.NoError(t, copyFile(filepath.Join("resources", "spec.yaml"), filepath.Join(temp, "spec.yaml")))
+
+	require.NoError(t, execute(t, temp, "run", "-t", "all", "--pinned", "--skip-compile").Run())
+	// check README.md exists
+	_, err = os.Stat(filepath.Join(temp, "README.md"))
+	require.NoError(t, err)
+}
+
+
 func TestGenerationWorkflows(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
@@ -94,7 +129,7 @@ func TestGenerationWorkflows(t *testing.T) {
 			workflowFile.Sources["first-source"] = workflow.Source{
 				Inputs: []workflow.Document{
 					{
-						Location: tt.inputDoc,
+						Location: workflow.LocationString(tt.inputDoc),
 					},
 				},
 			}
@@ -201,16 +236,16 @@ func execute(t *testing.T, wd string, args ...string) Runnable {
 	_, filename, _, _ := runtime.Caller(0)
 	baseFolder := filepath.Join(filepath.Dir(filename), "..")
 	mainGo := filepath.Join(baseFolder, "main.go")
-	cmd := exec.Command("go", append([]string{"run", mainGo}, args...)...)
-	cmd.Env = os.Environ()
-	cmd.Dir = wd
+	execCmd := exec.Command("go", append([]string{"run", mainGo}, args...)...)
+	execCmd.Env = os.Environ()
+	execCmd.Dir = wd
 	// store stdout and stderr in a buffer and output it all in one go if there's a failure
 	out := bytes.Buffer{}
-	cmd.Stdout = &out
-	cmd.Stderr = &out
+	execCmd.Stdout = &out
+	execCmd.Stderr = &out
 
 	return &subprocessRunner{
-		cmd: cmd,
+		cmd: execCmd,
 		out: &out,
 	}
 }
@@ -313,7 +348,7 @@ func TestSpecWorkflows(t *testing.T) {
 					require.NoError(t, err)
 				}
 				inputs = append(inputs, workflow.Document{
-					Location: inputDoc,
+					Location: workflow.LocationString(inputDoc),
 				})
 			}
 			var overlays []workflow.Overlay
@@ -324,7 +359,7 @@ func TestSpecWorkflows(t *testing.T) {
 				}
 				overlays = append(overlays, workflow.Overlay{
 					Document: &workflow.Document{
-						Location: overlay,
+						Location: workflow.LocationString(overlay),
 					},
 				})
 			}
@@ -424,7 +459,7 @@ func TestFallbackCodeSamplesWorkflow(t *testing.T) {
 		Version: workflow.WorkflowVersion,
 		Sources: map[string]workflow.Source{
 			"first-source": {
-				Inputs: []workflow.Document{{Location: relFilePath}},
+				Inputs: []workflow.Document{{Location: workflow.LocationString(relFilePath)}},
 				Overlays: []workflow.Overlay{
 					{
 						FallbackCodeSamples: &workflow.FallbackCodeSamples{

internal/download/download.go

@@ -302,7 +302,7 @@ func ResolveRemoteDocument(ctx context.Context, d workflow.Document, outPath str
 		token = os.Getenv(strings.ToUpper(envVar))
 	}
 
-	res, err := download.Fetch(d.Location, header, token)
+	res, err := download.Fetch(d.Location.Resolve(), header, token)
 	if err != nil {
 		return "", err
 	}

internal/migrate/migrate.go

@@ -334,7 +334,7 @@ func docLocationsToDocuments(docLocations []string, authHeader string, hasToken
 	var documents []workflow.Document
 	for _, docLocation := range docLocations {
 		documents = append(documents, workflow.Document{
-			Location: docLocation,
+			Location: workflow.LocationString(docLocation),
 			Auth:     auth,
 		})
 	}

internal/run/minimumViableSpec.go

@@ -26,7 +26,7 @@ func (w *Workflow) retryWithMinimumViableSpec(ctx context.Context, parentStep *w
 
 	substep := parentStep.NewSubstep("Retrying with minimum viable document")
 	source := w.workflow.Sources[sourceID]
-	baseLocation := source.Inputs[0].Location
+	baseLocation := source.Inputs[0].Location.Resolve()
 	workingDir := workflow.GetTempDir()
 
 	// This is intended to only be used from quickstart, we must assume a singular input document
@@ -65,7 +65,7 @@ func (w *Workflow) retryWithMinimumViableSpec(ctx context.Context, parentStep *w
 		}
 	}()
 
-	_, _, model, err := openapi.LoadDocument(ctx, source.Inputs[0].Location)
+	_, _, model, err := openapi.LoadDocument(ctx, source.Inputs[0].Location.Resolve())
 	if err != nil {
 		return "", nil, fmt.Errorf("failed to load document: %w", err)
 	}

internal/run/run.go

@@ -131,10 +131,6 @@ func (w *Workflow) RunInner(ctx context.Context) error {
 		return fmt.Errorf("cannot specify both a target and a source")
 	}
 
-	if w.SetVersion != "" && len(w.workflow.Targets) > 1 {
-		return fmt.Errorf("cannot manually apply a version when more than one target is specified ")
-	}
-
 	sourceIDs := []string{w.Source}
 	if w.Source == "all" {
 		sourceIDs = lo.Keys(w.workflow.Sources)
@@ -144,6 +140,10 @@ func (w *Workflow) RunInner(ctx context.Context) error {
 		targetIDs = lo.Keys(w.workflow.Targets)
 	}
 
+	if w.SetVersion != "" && len(targetIDs) > 1 {
+		return fmt.Errorf("cannot manually apply a version when more than one target is specified ")
+	}
+
 	for _, sourceID := range sourceIDs {
 		if sourceID == "" {
 			continue

internal/run/source.go

@@ -111,7 +111,7 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W
 		var orgSlug, workspaceSlug, registryNamespace string
 		if isSingleRegistrySource(w.workflow.Sources[sourceID]) && w.workflow.Sources[sourceID].Registry == nil {
 			d := w.workflow.Sources[sourceID].Inputs[0]
-			registryBreakdown := workflow.ParseSpeakeasyRegistryReference(d.Location)
+			registryBreakdown := workflow.ParseSpeakeasyRegistryReference(d.Location.Resolve())
 			if registryBreakdown == nil {
 				return "", nil, fmt.Errorf("failed to parse speakeasy registry reference %s", d.Location)
 			}
@@ -133,7 +133,7 @@ func (w *Workflow) RunSource(ctx context.Context, parentStep *workflowTracking.W
 		}
 		registryLocation := fmt.Sprintf("%s/%s/%s/%s@%s", "registry.speakeasyapi.dev", orgSlug, workspaceSlug,
 			lockSource.SourceNamespace, lockSource.SourceRevisionDigest)
-		d := workflow.Document{Location: registryLocation}
+		d := workflow.Document{Location: workflow.LocationString(registryLocation)}
 		docPath, err := registry.ResolveSpeakeasyRegistryBundle(ctx, d, workflow.GetTempDir())
 		if err != nil {
 			return "", nil, fmt.Errorf("error resolving registry bundle from %s: %w", registryLocation, err)
@@ -318,7 +318,7 @@ func (w *Workflow) computeChanges(ctx context.Context, rootStep *workflowTrackin
 
 	changesStep.NewSubstep("Downloading prior revision")
 
-	d := workflow.Document{Location: oldRegistryLocation}
+	d := workflow.Document{Location: workflow.LocationString(oldRegistryLocation)}
 	oldDocPath, err := registry.ResolveSpeakeasyRegistryBundle(ctx, d, workflow.GetTempDir())
 	if err != nil {
 		return

internal/schema/document.go

@@ -13,7 +13,7 @@ import (
 )
 
 func LoadDocument(ctx context.Context, schemaLocation string) ([]byte, *libopenapi.Document, *libopenapi.DocumentModel[v3.Document], error) {
-	docPath, err := ResolveDocument(ctx, workflow.Document{Location: schemaLocation}, nil, nil)
+	docPath, err := ResolveDocument(ctx, workflow.Document{Location: workflow.LocationString(schemaLocation)}, nil, nil)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -53,5 +53,5 @@ func ResolveDocument(ctx context.Context, d workflow.Document, outputLocation *s
 		return documentOut, nil
 	}
 
-	return d.Location, nil
+	return d.Location.Resolve(), nil
 }

internal/studio/modifications/overlay.go

@@ -91,10 +91,10 @@ func UpsertOverlayIntoSource(source *workflow.Source, overlayPath string) {
 	}
 
 	// Add the new overlay to the source, if not already present
-	if !slices.ContainsFunc(source.Overlays, func(o workflow.Overlay) bool { return o.Document.Location == overlayPath }) {
+	if !slices.ContainsFunc(source.Overlays, func(o workflow.Overlay) bool { return o.Document.Location.Reference() == overlayPath }) {
 		source.Overlays = append(source.Overlays, workflow.Overlay{
 			Document: &workflow.Document{
-				Location: overlayPath,
+				Location: workflow.LocationString(overlayPath),
 			},
 		})
 	}

internal/studio/studioHandlers.go

@@ -59,7 +59,7 @@ func NewStudioHandlers(ctx context.Context, workflowRunner *run.Workflow) (*Stud
 		// If there are multiple modifications overlays - we take the last one
 		contents, _ := isStudioModificationsOverlay(overlay)
 		if contents != "" {
-			ret.OverlayPath = overlay.Document.Location
+			ret.OverlayPath = overlay.Document.Location.Resolve()
 		}
 	}
 
@@ -230,7 +230,7 @@ func (h *StudioHandlers) updateSource(r *http.Request) error {
 			return errors.ErrBadRequest.Wrap(fmt.Errorf("cannot update source input to a remote file"))
 		}
 
-		inputLocation := source.Inputs[0].Location
+		inputLocation := source.Inputs[0].Location.Resolve()
 
 		// if it's absolute that's fine, otherwise it's relative to the project directory
 		if !filepath.IsAbs(inputLocation) {
@@ -580,14 +580,14 @@ func findWorkflowSourceIDBasedOnTarget(workflow run.Workflow, targetID string) (
 
 func isStudioModificationsOverlay(overlay workflow.Overlay) (string, error) {
 	isLocalFile := overlay.Document != nil &&
-		!strings.HasPrefix(overlay.Document.Location, "https://") &&
-		!strings.HasPrefix(overlay.Document.Location, "http://") &&
-		!strings.HasPrefix(overlay.Document.Location, "registry.speakeasyapi.dev")
+		!strings.HasPrefix(overlay.Document.Location.Resolve(), "https://") &&
+		!strings.HasPrefix(overlay.Document.Location.Resolve(), "http://") &&
+		!strings.HasPrefix(overlay.Document.Location.Resolve(), "registry.speakeasyapi.dev")
 	if !isLocalFile {
 		return "", nil
 	}
 
-	asString, err := utils.ReadFileToString(overlay.Document.Location)
+	asString, err := utils.ReadFileToString(overlay.Document.Location.Resolve())
 
 	if err != nil {
 		return "", err

prompts/configs.go

@@ -175,7 +175,7 @@ func setDevContainerDefaults(output *config.Configuration, wf *workflow.Workflow
 			if source.Output != nil {
 				schemaPath = *source.Output
 			} else {
-				schemaPath = source.Inputs[0].Location
+				schemaPath = source.Inputs[0].Location.Resolve()
 			}
 			output.Generation.DevContainers = &config.DevContainers{
 				Enabled:    true,

prompts/sources.go

@@ -267,9 +267,9 @@ func AddToSource(name string, currentSource *workflow.Source) (*workflow.Source,
 		}
 
 		for index, input := range currentSource.Inputs {
-			if input.Location == selectedDoc {
+			if input.Location.Reference() == selectedDoc {
 				newInput := workflow.Document{}
-				newInput.Location = fileLocation
+				newInput.Location = workflow.LocationString(fileLocation)
 				if authHeader != "" {
 					newInput.Auth = &workflow.Auth{
 						Header: authHeader,
@@ -441,7 +441,7 @@ func formatDocument(fileLocation, authHeader string, validate bool) (*workflow.D
 	}
 
 	document := &workflow.Document{
-		Location: fileLocation,
+		Location: workflow.LocationString(fileLocation),
 	}
 
 	if authHeader != "" {

prompts/utils.go

@@ -89,7 +89,7 @@ func getCurrentInputs(currentSource *workflow.Source) []string {
 	var sources []string
 	if currentSource != nil {
 		for _, input := range currentSource.Inputs {
-			sources = append(sources, input.Location)
+			sources = append(sources, input.Location.Reference())
 		}
 	}
 	return sources

registry/utils.go

@@ -26,7 +26,7 @@ func ResolveSpeakeasyRegistryBundle(ctx context.Context, d workflow.Document, ou
 		return nil, fmt.Errorf("unable to use speakeasy registry reference without authenticating")
 	}
 
-	registryBreakdown := workflow.ParseSpeakeasyRegistryReference(d.Location)
+	registryBreakdown := workflow.ParseSpeakeasyRegistryReference(d.Location.Resolve())
 	if registryBreakdown == nil {
 		return nil, fmt.Errorf("failed to parse speakeasy registry reference %s", d.Location)
 	}