From 29a1df5ab0f4b2affe5ebcbe50b61d10c914f9ee Mon Sep 17 00:00:00 2001 From: gabriel-farache Date: Thu, 11 Jul 2024 10:05:17 +0200 Subject: [PATCH] feat: add query parameters to dump CLI log file into stdout (#166) Signed-off-by: gabriel-farache --- assets/openapi.json | 20 ++++++++ internal/filesystem/filesystem.go | 57 ++++++++++++++++------ internal/filesystem/interface.go | 4 +- internal/move2kubeapi/handlers/handlers.go | 2 + internal/move2kubeapi/handlers/outputs.go | 3 +- internal/move2kubeapi/handlers/plan.go | 3 +- 6 files changed, 69 insertions(+), 20 deletions(-) diff --git a/assets/openapi.json b/assets/openapi.json index a4ecb3d..ab52183 100755 --- a/assets/openapi.json +++ b/assets/openapi.json @@ -1101,6 +1101,16 @@ "schema": { "$ref": "#/components/schemas/RemoteSource" } + }, + { + "name": "dump-cli-logs", + "in": "query", + "description": "Boolean to dump the CLI logs content in stdout.", + "required": false, + "example": "true", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -1333,6 +1343,16 @@ "schema": { "type": "boolean" } + }, + { + "name": "dump-cli-logs", + "in": "query", + "description": "Boolean to dump the CLI logs content in stdout.", + "required": false, + "example": "true", + "schema": { + "type": "boolean" + } } ], "requestBody": { diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go index 4844f78..33fd1df 100644 --- a/internal/filesystem/filesystem.go +++ b/internal/filesystem/filesystem.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net" @@ -1172,7 +1173,7 @@ func (fs *FileSystem) deleteProjectInput(t *bolt.Tx, workspaceId, projectId, pro // StartPlanning starts the generation of a plan for a project. // If plan generation is ongoing it will return an error. -func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error { +func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool, dumpCliLogs bool) error { logrus.Trace("FileSystem.StartPlanning start") defer logrus.Trace("FileSystem.StartPlanning end") db, err := fs.GetDatabase(false) @@ -1182,11 +1183,11 @@ func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, } defer db.Close() return db.Update(func(t *bolt.Tx) error { - return fs.startPlanning(t, workspaceId, projectId, remoteSource, debugMode) + return fs.startPlanning(t, workspaceId, projectId, remoteSource, debugMode, dumpCliLogs) }) } -func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSource string, debugMode bool) error { +func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSource string, debugMode bool, dumpCliLogs bool) error { logrus.Trace("FileSystem.startPlanning start") defer logrus.Trace("FileSystem.startPlanning end") // check conditions @@ -1345,7 +1346,7 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSo } // start plan generation logrus.Debugf("just before starting planning for project %s in workspace %s", projectId, workspaceId) - go fs.runPlan(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunDir, message, planProgressServerMeta.Port, workspaceId, projectId, project.Name, debugMode) + go fs.runPlan(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunDir, message, planProgressServerMeta.Port, workspaceId, projectId, project.Name, debugMode, dumpCliLogs) logrus.Infof("Planning started for the project with id %s", projectId) return nil } @@ -1617,23 +1618,23 @@ func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, p currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...) } // resume the transformation - go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, true) + go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, true, true) return nil } // StartTransformation starts the transformation for a project. -func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error { +func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool, dumpCliLogs bool) error { db, err := fs.GetDatabase(false) if err != nil { return err } defer db.Close() return db.Update(func(t *bolt.Tx) error { - return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode, skipQA) + return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode, skipQA, dumpCliLogs) }) } -func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error { +func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool, dumpCliLogs bool) error { // check conditions project, err := fs.readProject(t, workspaceId, projectId) if err != nil { @@ -1811,7 +1812,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...) } // start the transformation - go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, false) + go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, false, dumpCliLogs) logrus.Infof("Waiting for QA engine to start for the output '%s' of the project '%s'", projOutput.Id, projectId) if err := <-transformCh; err != nil { return fmt.Errorf("failed to start the transformation and qa engine. Error: %w", err) @@ -2216,15 +2217,16 @@ func validateAndProcessPlan(plan string, shouldProcess bool) (string, error) { } // runPlan starts the planning. -func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, workspaceId, projectId, projectName string, debugMode bool) error { +func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, workspaceId, projectId, projectName string, debugMode bool, dumpCliLogs bool) error { logrus.Trace("FileSystem.runPlan start") defer logrus.Trace("FileSystem.runPlan end") - logrus.Infof("Starting plan at directory %s using configs %+v and source %s and customizations %s to output %s", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir) + logrus.Infof("Starting plan at directory %s using configs %+v and source %s and customizations %s to output %s. Will dump cli logs to stdout: %t", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, dumpCliLogs) normName, err := common.NormalizeName(projectName) if err != nil { return types.ErrorValidation{Reason: fmt.Sprintf("failed to normalize the project name %s . Error: %q", projectName, err)} } - cmdArgs := []string{"plan", "--name", normName, "--plan-progress-port", cast.ToString(port), "--log-file", M2K_CLI_LOG_FILE} + logFile := M2K_CLI_LOG_FILE + cmdArgs := []string{"plan", "--name", normName, "--plan-progress-port", cast.ToString(port), "--log-file", logFile} verbose := debugMode || isVerbose() if currentRunSrcDir != "" { cmdArgs = append(cmdArgs, "--source", currentRunSrcDir) @@ -2313,6 +2315,9 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri default: logrus.Debug("ctx not closed") } + if dumpCliLogs { + dumpLogFileToStdOut(currentRunDir, logFile) + } // release lock db, err := fs.GetDatabase(false) if err != nil { @@ -2375,13 +2380,14 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri return err } -func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, skipQA bool, overwriteOutDir bool) error { - logrus.Infof("Starting transformation in %s with configs from %+v and source from %s , customizations from %s and output to %s", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir) +func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, skipQA bool, overwriteOutDir bool, dumpCliLogs bool) error { + logrus.Infof("Starting transformation in %s with configs from %+v and source from %s , customizations from %s and output to %s. Will dump cli logs to stdout: %t", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, dumpCliLogs) portStr, err := cast.ToStringE(port) if err != nil { return fmt.Errorf("failed to convert the port '%d' to a string. Error: %q", port, err) } - cmdArgs := []string{"transform", "--qa-disable-cli", "--qa-port", portStr, "--output", currentRunOutDir, "--log-file", M2K_CLI_LOG_FILE} + logFile := M2K_CLI_LOG_FILE + cmdArgs := []string{"transform", "--qa-disable-cli", "--qa-port", portStr, "--output", currentRunOutDir, "--log-file", logFile} if currentRunSrcDir != "" { cmdArgs = append(cmdArgs, "--source", currentRunSrcDir) } @@ -2495,7 +2501,12 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [ default: logrus.Debug("ctx not closed") } - // create the output zip file + if dumpCliLogs { + err = dumpLogFileToStdOut(currentRunDir, logFile) + if err != nil { + logrus.Errorf("failed to dump the CLI log file to stdout. Error: %q", err) + } + } // create the output zip file if err := copyOverPlanConfigAndQACache(currentRunDir, currentRunOutDir); err != nil { logrus.Errorf("failed to copy over the m2kconfig.yaml and m2kqacache.yaml. Error: %q", err) } @@ -2754,3 +2765,17 @@ func (s mySortable) Swap(i, j int) { s.times[i], s.times[j] = s.times[j], s.times[i] s.ids[i], s.ids[j] = s.ids[j], s.ids[i] } + +func dumpLogFileToStdOut(workdir string, logFileName string) error { + logFilePath := filepath.Join(workdir, logFileName) + if _, err := os.Stat(logFilePath); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("log file %q does not exists", logFilePath) + } + contentBytes, err := os.ReadFile(logFilePath) + if err != nil { + return fmt.Errorf("error while reading log file %q: %w", logFilePath, err) + } + logrus.Infof("Logs:\n%s", string(contentBytes)) + + return nil +} diff --git a/internal/filesystem/interface.go b/internal/filesystem/interface.go index 59a4661..7910edc 100644 --- a/internal/filesystem/interface.go +++ b/internal/filesystem/interface.go @@ -40,11 +40,11 @@ type IFileSystem interface { CreateProjectInput(workspaceId, projectId string, projInput types.ProjectInput, file io.Reader, isCommon bool) error ReadProjectInput(workspaceId, projectId, projInputId string, isCommon bool) (projInput types.ProjectInput, file io.Reader, err error) DeleteProjectInput(workspaceId, projectId, projInputId string, isCommon bool) error - StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error + StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool, dumpCliLogs bool) error ReadPlan(workspaceId, projectId string) (plan io.Reader, err error) UpdatePlan(workspaceId, projectId string, plan io.Reader) error DeletePlan(workspaceId, projectId string) error - StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error + StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool, dumpCliLogs bool) error ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error ReadProjectOutput(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error) ReadProjectOutputGraph(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error) diff --git a/internal/move2kubeapi/handlers/handlers.go b/internal/move2kubeapi/handlers/handlers.go index e5f4bbe..0f698fd 100644 --- a/internal/move2kubeapi/handlers/handlers.go +++ b/internal/move2kubeapi/handlers/handlers.go @@ -30,6 +30,8 @@ import ( ) const ( + // DUMP_CLI_LOGS_STDOUT_PARAM is the name of the query parameter used for dump the content of CLI log file into stdout + DUMP_CLI_LOGS_STDOUT_PARAM = "dump-cli-logs" // SKIP_QA_QUERY_PARAM is the name of the query parameter used for skipping QA SKIP_QA_QUERY_PARAM = "skip-qa" // REMOTE_SOURCE_QUERY_PARAM is the URL of the git remote to be used as source diff --git a/internal/move2kubeapi/handlers/outputs.go b/internal/move2kubeapi/handlers/outputs.go index 29ee4ab..c3c7aaf 100644 --- a/internal/move2kubeapi/handlers/outputs.go +++ b/internal/move2kubeapi/handlers/outputs.go @@ -55,6 +55,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) { } debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true" skipQA := r.URL.Query().Get(SKIP_QA_QUERY_PARAM) == "true" + dumpCliLogs := r.URL.Query().Get(DUMP_CLI_LOGS_STDOUT_PARAM) == "true" timestamp, _, err := common.GetTimestamp() if err != nil { logrus.Errorf("failed to get the timestamp. Error: %q", err) @@ -66,7 +67,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) { projOutput.Timestamp = timestamp projOutput.Name = projOutput.Id // This isn't really used anywhere projOutput.Status = types.ProjectOutputStatusInProgress - if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode, skipQA); err != nil { + if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode, skipQA, dumpCliLogs); err != nil { logrus.Errorf("failed to start the transformation. Error: %q", err) if notExErr, ok := err.(types.ErrorDoesNotExist); ok { if notExErr.Id == "plan" { diff --git a/internal/move2kubeapi/handlers/plan.go b/internal/move2kubeapi/handlers/plan.go index 9fbc004..eee57a4 100644 --- a/internal/move2kubeapi/handlers/plan.go +++ b/internal/move2kubeapi/handlers/plan.go @@ -46,7 +46,8 @@ func HandleStartPlanning(w http.ResponseWriter, r *http.Request) { return } debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true" - if err := m2kFS.StartPlanning(workspaceId, projectId, remoteSource, debugMode); err != nil { + dumpCliLogs := r.URL.Query().Get(DUMP_CLI_LOGS_STDOUT_PARAM) == "true" + if err := m2kFS.StartPlanning(workspaceId, projectId, remoteSource, debugMode, dumpCliLogs); err != nil { logrus.Errorf("failed to start plan generation. Error: %q", err) if _, ok := err.(types.ErrorDoesNotExist); ok { w.WriteHeader(http.StatusNotFound)