feat(api): add source and qa-skip as parameters for plan and outputs …

…endpoints (#155)

* Add source and skipQA  in REST API endpoint query parameter
* Improving how transformation cmd completion is detected

Signed-off-by: gabriel-farache <gfarache@redhat.com>

assets/openapi.json

@@ -1091,6 +1091,16 @@
             "schema": {
               "$ref": "#/components/schemas/ID"
             }
+          },
+          {
+            "name": "remote-source",
+            "in": "query",
+            "description": "Remote source git URL from which get the source files.",
+            "required": false,
+            "example": "git+https://github.com/konveyor/move2kube",
+            "schema": {
+              "$ref": "#/components/schemas/RemoteSource"
+            }
           }
         ],
         "responses": {
@@ -1313,6 +1323,16 @@
             "schema": {
               "$ref": "#/components/schemas/ID"
             }
+          },
+          {
+            "name": "skip-qa",
+            "in": "query",
+            "description": "Boolean to skip interactive QA.",
+            "required": false,
+            "example": "true",
+            "schema": {
+              "type": "boolean"
+            }
           }
         ],
         "requestBody": {
@@ -2295,6 +2315,12 @@
         "description": "A unique ID.",
         "example": "id-1234"
       },
+      "RemoteSource": {
+        "pattern": "^git[+](https|ssh)://[a-zA-Z0-9]+([-.]{1}[a-zA-Z0-9]+)*[.][a-zA-Z]{2,5}(:[0-9]{1,5})?(\/.*)?$",
+        "type": "string",
+        "description": "A git URL.",
+        "example": "git+https://github.com/konveyor/move2kube"
+      },
       "Error": {
         "required": [
           "error"

internal/common/constants.go

@@ -65,6 +65,8 @@ var (
 	AuthServerClient gocloak.GoCloak
 	// ID_REGEXP is the regexp used to check if a Id is valid
 	ID_REGEXP = regexp.MustCompile("^[a-zA-Z0-9-_]+$")
+	// REMOTE_SOURCE_REGEXP is the regexp used to check if a remote source is valid
+	REMOTE_SOURCE_REGEXP = regexp.MustCompile(`^git\+(https|ssh)://[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,5}(:[0-9]{1,5})?(\/.*)?$`)
 	// INVALID_NAME_CHARS_REGEXP is the regexp used to replace invalid name characters with hyphen
 	INVALID_NAME_CHARS_REGEXP = regexp.MustCompile("[^a-z0-9-]")
 	// AUTHZ_HEADER is the authorization header

internal/common/utils.go

@@ -123,6 +123,11 @@ func IsValidId(id string) bool {
 	return ID_REGEXP.MatchString(id)
 }
 
+// IsRemoteSource returns true if the provided remoteSource is valid
+func IsRemoteSource(remoteSource string) bool {
+	return REMOTE_SOURCE_REGEXP.MatchString(remoteSource)
+}
+
 // IsStringPresent checks if a value is present in a slice
 func IsStringPresent(list []string, value string) bool {
 	for _, val := range list {

internal/filesystem/filesystem.go

@@ -1172,7 +1172,7 @@ func (fs *FileSystem) deleteProjectInput(t *bolt.Tx, workspaceId, projectId, pro
 
 // StartPlanning starts the generation of a plan for a project.
 // If plan generation is ongoing it will return an error.
-func (fs *FileSystem) StartPlanning(workspaceId, projectId string, debugMode bool) error {
+func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error {
 	logrus.Trace("FileSystem.StartPlanning start")
 	defer logrus.Trace("FileSystem.StartPlanning end")
 	db, err := fs.GetDatabase(false)
@@ -1182,11 +1182,11 @@ func (fs *FileSystem) StartPlanning(workspaceId, projectId string, debugMode boo
 	}
 	defer db.Close()
 	return db.Update(func(t *bolt.Tx) error {
-		return fs.startPlanning(t, workspaceId, projectId, debugMode)
+		return fs.startPlanning(t, workspaceId, projectId, remoteSource, debugMode)
 	})
 }
 
-func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, debugMode bool) error {
+func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSource string, debugMode bool) error {
 	logrus.Trace("FileSystem.startPlanning start")
 	defer logrus.Trace("FileSystem.startPlanning end")
 	// check conditions
@@ -1198,7 +1198,7 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, d
 		return types.ErrorOngoing{Id: projectId}
 	}
 
-	if !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] {
+	if remoteSource == "" && !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] {
 		if !project.Status[types.ProjectStatusInputReference] {
 			return types.ErrorValidation{Reason: "the project has no source folders or customization folders as input"}
 		}
@@ -1261,8 +1261,8 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, d
 		return fmt.Errorf("failed to resolve the temporary directory '%s' as a symbolic link. Error: %w", currentRunDir, err)
 	}
 	// default is empty string, if the input source is given, value is updated.
-	currentRunSrcDir := ""
-	if project.Status[types.ProjectStatusInputSources] {
+	currentRunSrcDir := remoteSource
+	if currentRunSrcDir == "" && project.Status[types.ProjectStatusInputSources] {
 		currentRunSrcDir = filepath.Join(currentRunDir, SOURCES_DIR)
 		currentRunSrcDirSrc := filepath.Join(projInputsDir, EXPANDED_DIR, SOURCES_DIR)
 		if err := copyDir(currentRunSrcDirSrc, currentRunSrcDir); err != nil {
@@ -1305,7 +1305,7 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, d
 			inpPathSrc := ""
 			inpPathDst := ""
 			workInp := work.Inputs[inp.Id]
-			if workInp.Type == types.ProjectInputSources {
+			if workInp.Type == types.ProjectInputSources && remoteSource != "" {
 				if currentRunSrcDir == "" {
 					currentRunSrcDir = filepath.Join(currentRunDir, SOURCES_DIR)
 				}
@@ -1515,18 +1515,18 @@ func (fs *FileSystem) deletePlan(t *bolt.Tx, workspaceId, projectId string) erro
 }
 
 // ResumeTransformation resumes a transformation that did not finish
-func (fs *FileSystem) ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode bool) error {
+func (fs *FileSystem) ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error {
 	db, err := fs.GetDatabase(false)
 	if err != nil {
 		return err
 	}
 	defer db.Close()
 	return db.Update(func(t *bolt.Tx) error {
-		return fs.resumeTransformation(t, workspaceId, projectId, projOutputId, debugMode)
+		return fs.resumeTransformation(t, workspaceId, projectId, projOutputId, debugMode, skipQA)
 	})
 }
 
-func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, projOutputId string, debugMode bool) error {
+func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error {
 	// check conditions
 	project, err := fs.readProject(t, workspaceId, projectId)
 	if err != nil {
@@ -1617,23 +1617,23 @@ func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, p
 		currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...)
 	}
 	// resume the transformation
-	go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, true)
+	go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, true)
 	return nil
 }
 
 // StartTransformation starts the transformation for a project.
-func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode bool) error {
+func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error {
 	db, err := fs.GetDatabase(false)
 	if err != nil {
 		return err
 	}
 	defer db.Close()
 	return db.Update(func(t *bolt.Tx) error {
-		return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode)
+		return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode, skipQA)
 	})
 }
 
-func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode bool) error {
+func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error {
 	// check conditions
 	project, err := fs.readProject(t, workspaceId, projectId)
 	if err != nil {
@@ -1642,7 +1642,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str
 	if _, ok := project.Outputs[projOutput.Id]; ok {
 		return types.ErrorIdAlreadyInUse{Id: projOutput.Id}
 	}
-	if !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] {
+	if !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] && !project.Status[types.ProjectStatusRemoteInputSources] {
 		if !project.Status[types.ProjectStatusInputReference] {
 			return types.ErrorValidation{Reason: "the project has no source or customization folders as input"}
 		}
@@ -1727,7 +1727,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str
 	// default is empty string, if the input source is given, the value is updated
 	currentRunSrcDir := ""
 	// copy the source and customizations directories into the run directory
-	if project.Status[types.ProjectStatusInputSources] {
+	if project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusRemoteInputSources] {
 		currentRunSrcDir = filepath.Join(currentRunDir, SOURCES_DIR)
 		srcPath := filepath.Join(projInputsDir, EXPANDED_DIR, SOURCES_DIR)
 		if err := copyDir(srcPath, currentRunSrcDir); err != nil {
@@ -1811,7 +1811,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str
 		currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...)
 	}
 	// start the transformation
-	go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, false)
+	go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, false)
 	logrus.Infof("Waiting for QA engine to start for the output '%s' of the project '%s'", projOutput.Id, projectId)
 	if err := <-transformCh; err != nil {
 		return fmt.Errorf("failed to start the transformation and qa engine. Error: %w", err)
@@ -2156,7 +2156,7 @@ func NewFileSystem() (*FileSystem, error) {
 		}
 		for _, project := range projects {
 			for _, projOutput := range project.Outputs {
-				if err := fileSystem.ResumeTransformation(workspace.Id, project.Id, projOutput.Id, false); err != nil {
+				if err := fileSystem.ResumeTransformation(workspace.Id, project.Id, projOutput.Id, false, false); err != nil {
 					logrus.Debugf("failed to resume the transformation for output with id: %s of project id: %s . Error: %q", projOutput.Id, project.Id, err)
 				}
 			}
@@ -2206,8 +2206,8 @@ func validateAndProcessPlan(plan string, shouldProcess bool) (string, error) {
 		return "", fmt.Errorf("'spec.sourceDir' is missing from the plan")
 	} else if pSpecSourceDir, ok := pSpecSourceDirI.(string); !ok {
 		return "", fmt.Errorf("'spec.sourceDir' is not a string. Actual value is %+v of type %T", pSpecSourceDirI, pSpecSourceDirI)
-	} else if pSpecSourceDir != SOURCES_DIR && pSpecSourceDir != "" {
-		return "", fmt.Errorf("'spec.sourceDir' is invalid. Expected 'source' . Actual: %s", pSpecSourceDir)
+	} else if pSpecSourceDir != SOURCES_DIR && pSpecSourceDir != "" && !strings.HasPrefix(pSpecSourceDir, "git+https://") {
+		return "", fmt.Errorf("'spec.sourceDir' is invalid. Expected 'source' or 'git+https://<remote repo url> . Actual: %s", pSpecSourceDir)
 	} else {
 		// TODO: better processing of the plan
 		pMeta["name"], _ = common.NormalizeName(pMetaName)
@@ -2333,6 +2333,9 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri
 		}
 		// update state
 		logrus.Debug("planning finished. inside Update. just before update start")
+		if strings.HasPrefix(currentRunSrcDir, "git+https://") {
+			project.Status[types.ProjectStatusRemoteInputSources] = true
+		}
 		project.Status[types.ProjectStatusPlanning] = false
 		project.Status[types.ProjectStatusPlan] = true
 		project.Status[types.ProjectStatusStalePlan] = false
@@ -2372,7 +2375,7 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri
 	return err
 }
 
-func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, overwriteOutDir bool) error {
+func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, skipQA bool, overwriteOutDir bool) error {
 	logrus.Infof("Starting transformation in %s with configs from %+v and source from %s , customizations from %s and output to %s", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir)
 	portStr, err := cast.ToStringE(port)
 	if err != nil {
@@ -2389,6 +2392,9 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [
 	if verbose {
 		cmdArgs = append(cmdArgs, "--log-level", "trace")
 	}
+	if skipQA {
+		cmdArgs = append(cmdArgs, "--qa-skip")
+	}
 	if !common.Config.EnableLocalExecution {
 		cmdArgs = append(cmdArgs, "--disable-local-execution")
 	}
@@ -2421,6 +2427,19 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [
 		logrus.Errorf("failed to start the transform command. Error: %q", err)
 		return err
 	}
+	flag := true
+	go func() {
+		if err := cmd.Wait(); err != nil {
+			logrus.Errorf("failed to wait for end of the transform command. Error: %q", err)
+		}
+		logrus.Debugf("Closing transformCh: %t", flag)
+		if flag {
+			flag = false
+			transformCh <- nil
+			close(transformCh)
+		}
+
+	}()
 	wg := sync.WaitGroup{}
 	outCh := make(chan string, 10)
 	stdoutReader := bufio.NewReader(stdout)
@@ -2448,17 +2467,17 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [
 			}
 			text, err = stderrReader.ReadString('\n')
 		}
-		logrus.Debugf("failed to fetch the stderr of move2kube transform. Error: %q", err)
+		logrus.Infof("failed to fetch the stderr of move2kube transform. Error: %q", err)
 		wg.Done()
 	}()
 	go func() {
 		wg.Wait()
 		close(outCh)
 	}()
-	flag := true
 	for outputLine := range outCh {
 		if flag && strings.Contains(outputLine, portStr) {
 			flag = false
+			logrus.Debug("Closing transformCh")
 			transformCh <- nil
 			close(transformCh)
 		}

internal/filesystem/interface.go

@@ -40,12 +40,12 @@ type IFileSystem interface {
 	CreateProjectInput(workspaceId, projectId string, projInput types.ProjectInput, file io.Reader, isCommon bool) error
 	ReadProjectInput(workspaceId, projectId, projInputId string, isCommon bool) (projInput types.ProjectInput, file io.Reader, err error)
 	DeleteProjectInput(workspaceId, projectId, projInputId string, isCommon bool) error
-	StartPlanning(workspaceId, projectId string, debugMode bool) error
+	StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error
 	ReadPlan(workspaceId, projectId string) (plan io.Reader, err error)
 	UpdatePlan(workspaceId, projectId string, plan io.Reader) error
 	DeletePlan(workspaceId, projectId string) error
-	StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode bool) error
-	ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode bool) error
+	StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error
+	ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error
 	ReadProjectOutput(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error)
 	ReadProjectOutputGraph(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error)
 	DeleteProjectOutput(workspaceId, projectId, projOutputId string) error

internal/move2kubeapi/handlers/handlers.go

@@ -30,6 +30,10 @@ import (
 )
 
 const (
+	// SKIP_QA_QUERY_PARAM is the name of the query parameter used for skipping QA
+	SKIP_QA_QUERY_PARAM = "skip-qa"
+	// REMOTE_SOURCE_QUERY_PARAM is the URL of the git remote to be used as source
+	REMOTE_SOURCE_QUERY_PARAM = "remote-source"
 	// DEBUG_QUERY_PARAM is the name of the query parameter used for debug mode
 	DEBUG_QUERY_PARAM = "debug"
 	// WORKSPACE_ID_ROUTE_VAR is the route variable that contains the workspace Id

internal/move2kubeapi/handlers/outputs.go

@@ -54,6 +54,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) {
 		planReader = nil
 	}
 	debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true"
+	skipQA := r.URL.Query().Get(SKIP_QA_QUERY_PARAM) == "true"
 	timestamp, _, err := common.GetTimestamp()
 	if err != nil {
 		logrus.Errorf("failed to get the timestamp. Error: %q", err)
@@ -65,7 +66,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) {
 	projOutput.Timestamp = timestamp
 	projOutput.Name = projOutput.Id // This isn't really used anywhere
 	projOutput.Status = types.ProjectOutputStatusInProgress
-	if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode); err != nil {
+	if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode, skipQA); err != nil {
 		logrus.Errorf("failed to start the transformation. Error: %q", err)
 		if notExErr, ok := err.(types.ErrorDoesNotExist); ok {
 			if notExErr.Id == "plan" {

internal/move2kubeapi/handlers/plan.go

@@ -39,8 +39,14 @@ func HandleStartPlanning(w http.ResponseWriter, r *http.Request) {
 		sendErrorJSON(w, "invalid id", http.StatusBadRequest)
 		return
 	}
+	remoteSource := r.URL.Query().Get(REMOTE_SOURCE_QUERY_PARAM)
+	if remoteSource != "" && !common.IsRemoteSource(remoteSource) {
+		logrus.Errorf("invalid remote source format; not matching regexp %s. Actual: %s", common.REMOTE_SOURCE_REGEXP, remoteSource)
+		sendErrorJSON(w, "invalid remote source format", http.StatusBadRequest)
+		return
+	}
 	debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true"
-	if err := m2kFS.StartPlanning(workspaceId, projectId, debugMode); err != nil {
+	if err := m2kFS.StartPlanning(workspaceId, projectId, remoteSource, debugMode); err != nil {
 		logrus.Errorf("failed to start plan generation. Error: %q", err)
 		if _, ok := err.(types.ErrorDoesNotExist); ok {
 			w.WriteHeader(http.StatusNotFound)

internal/types/types.go

@@ -346,6 +346,8 @@ func (e ErrorTokenUnverifiable) Error() string {
 type ProjectStatus string
 
 const (
+	// ProjectStatusRemoteInputSources indicates the project has a remlote git source
+	ProjectStatusRemoteInputSources ProjectStatus = "remote"
 	// ProjectStatusInputSources indicates the project has source folder uploaded
 	ProjectStatusInputSources ProjectStatus = "sources"
 	// ProjectStatusInputCustomizations indicates the project has customizations folder uploaded