Skip to content

Commit

Permalink
feat: add query parameters to dump CLI log file into stdout (#166)
Browse files Browse the repository at this point in the history 
Signed-off-by: gabriel-farache <gfarache@redhat.com>
  • Loading branch information
@gabriel-farache
gabriel-farache authored Jul 11, 2024
1 parent 10ededa commit 29a1df5
Show file tree
Hide file tree
Showing 6 changed files with 69 additions and 20 deletions.
20 changes: 20 additions & 0 deletions assets/openapi.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1101,6 +1101,16 @@
"schema": {
"$ref": "#/components/schemas/RemoteSource"
}
},
{
"name": "dump-cli-logs",
"in": "query",
"description": "Boolean to dump the CLI logs content in stdout.",
"required": false,
"example": "true",
"schema": {
"type": "boolean"
}
}
],
"responses": {
Expand Down Expand Up @@ -1333,6 +1343,16 @@
"schema": {
"type": "boolean"
}
},
{
"name": "dump-cli-logs",
"in": "query",
"description": "Boolean to dump the CLI logs content in stdout.",
"required": false,
"example": "true",
"schema": {
"type": "boolean"
}
}
],
"requestBody": {
Expand Down
57 changes: 41 additions & 16 deletions internal/filesystem/filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
Expand Down Expand Up @@ -1172,7 +1173,7 @@ func (fs *FileSystem) deleteProjectInput(t *bolt.Tx, workspaceId, projectId, pro

// StartPlanning starts the generation of a plan for a project.
// If plan generation is ongoing it will return an error.
func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error {
func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool, dumpCliLogs bool) error {
logrus.Trace("FileSystem.StartPlanning start")
defer logrus.Trace("FileSystem.StartPlanning end")
db, err := fs.GetDatabase(false)
Expand All @@ -1182,11 +1183,11 @@ func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string,
}
defer db.Close()
return db.Update(func(t *bolt.Tx) error {
return fs.startPlanning(t, workspaceId, projectId, remoteSource, debugMode)
return fs.startPlanning(t, workspaceId, projectId, remoteSource, debugMode, dumpCliLogs)
})
}

func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSource string, debugMode bool) error {
func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSource string, debugMode bool, dumpCliLogs bool) error {
logrus.Trace("FileSystem.startPlanning start")
defer logrus.Trace("FileSystem.startPlanning end")
// check conditions
Expand Down Expand Up @@ -1345,7 +1346,7 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSo
}
// start plan generation
logrus.Debugf("just before starting planning for project %s in workspace %s", projectId, workspaceId)
go fs.runPlan(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunDir, message, planProgressServerMeta.Port, workspaceId, projectId, project.Name, debugMode)
go fs.runPlan(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunDir, message, planProgressServerMeta.Port, workspaceId, projectId, project.Name, debugMode, dumpCliLogs)
logrus.Infof("Planning started for the project with id %s", projectId)
return nil
}
Expand Down Expand Up @@ -1617,23 +1618,23 @@ func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, p
currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...)
}
// resume the transformation
go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, true)
go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, true, true)
return nil
}

// StartTransformation starts the transformation for a project.
func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error {
func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool, dumpCliLogs bool) error {
db, err := fs.GetDatabase(false)
if err != nil {
return err
}
defer db.Close()
return db.Update(func(t *bolt.Tx) error {
return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode, skipQA)
return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode, skipQA, dumpCliLogs)
})
}

func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error {
func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool, dumpCliLogs bool) error {
// check conditions
project, err := fs.readProject(t, workspaceId, projectId)
if err != nil {
Expand Down Expand Up @@ -1811,7 +1812,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str
currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...)
}
// start the transformation
go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, false)
go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, false, dumpCliLogs)
logrus.Infof("Waiting for QA engine to start for the output '%s' of the project '%s'", projOutput.Id, projectId)
if err := <-transformCh; err != nil {
return fmt.Errorf("failed to start the transformation and qa engine. Error: %w", err)
Expand Down Expand Up @@ -2216,15 +2217,16 @@ func validateAndProcessPlan(plan string, shouldProcess bool) (string, error) {
}

// runPlan starts the planning.
func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, workspaceId, projectId, projectName string, debugMode bool) error {
func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, workspaceId, projectId, projectName string, debugMode bool, dumpCliLogs bool) error {
logrus.Trace("FileSystem.runPlan start")
defer logrus.Trace("FileSystem.runPlan end")
logrus.Infof("Starting plan at directory %s using configs %+v and source %s and customizations %s to output %s", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir)
logrus.Infof("Starting plan at directory %s using configs %+v and source %s and customizations %s to output %s. Will dump cli logs to stdout: %t", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, dumpCliLogs)
normName, err := common.NormalizeName(projectName)
if err != nil {
return types.ErrorValidation{Reason: fmt.Sprintf("failed to normalize the project name %s . Error: %q", projectName, err)}
}
cmdArgs := []string{"plan", "--name", normName, "--plan-progress-port", cast.ToString(port), "--log-file", M2K_CLI_LOG_FILE}
logFile := M2K_CLI_LOG_FILE
cmdArgs := []string{"plan", "--name", normName, "--plan-progress-port", cast.ToString(port), "--log-file", logFile}
verbose := debugMode || isVerbose()
if currentRunSrcDir != "" {
cmdArgs = append(cmdArgs, "--source", currentRunSrcDir)
Expand Down Expand Up @@ -2313,6 +2315,9 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri
default:
logrus.Debug("ctx not closed")
}
if dumpCliLogs {
dumpLogFileToStdOut(currentRunDir, logFile)
}
// release lock
db, err := fs.GetDatabase(false)
if err != nil {
Expand Down Expand Up @@ -2375,13 +2380,14 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri
return err
}

func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, skipQA bool, overwriteOutDir bool) error {
logrus.Infof("Starting transformation in %s with configs from %+v and source from %s , customizations from %s and output to %s", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir)
func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, skipQA bool, overwriteOutDir bool, dumpCliLogs bool) error {
logrus.Infof("Starting transformation in %s with configs from %+v and source from %s , customizations from %s and output to %s. Will dump cli logs to stdout: %t", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, dumpCliLogs)
portStr, err := cast.ToStringE(port)
if err != nil {
return fmt.Errorf("failed to convert the port '%d' to a string. Error: %q", port, err)
}
cmdArgs := []string{"transform", "--qa-disable-cli", "--qa-port", portStr, "--output", currentRunOutDir, "--log-file", M2K_CLI_LOG_FILE}
logFile := M2K_CLI_LOG_FILE
cmdArgs := []string{"transform", "--qa-disable-cli", "--qa-port", portStr, "--output", currentRunOutDir, "--log-file", logFile}
if currentRunSrcDir != "" {
cmdArgs = append(cmdArgs, "--source", currentRunSrcDir)
}
Expand Down Expand Up @@ -2495,7 +2501,12 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [
default:
logrus.Debug("ctx not closed")
}
// create the output zip file
if dumpCliLogs {
err = dumpLogFileToStdOut(currentRunDir, logFile)
if err != nil {
logrus.Errorf("failed to dump the CLI log file to stdout. Error: %q", err)
}
} // create the output zip file
if err := copyOverPlanConfigAndQACache(currentRunDir, currentRunOutDir); err != nil {
logrus.Errorf("failed to copy over the m2kconfig.yaml and m2kqacache.yaml. Error: %q", err)
}
Expand Down Expand Up @@ -2754,3 +2765,17 @@ func (s mySortable) Swap(i, j int) {
s.times[i], s.times[j] = s.times[j], s.times[i]
s.ids[i], s.ids[j] = s.ids[j], s.ids[i]
}

func dumpLogFileToStdOut(workdir string, logFileName string) error {
logFilePath := filepath.Join(workdir, logFileName)
if _, err := os.Stat(logFilePath); errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("log file %q does not exists", logFilePath)
}
contentBytes, err := os.ReadFile(logFilePath)
if err != nil {
return fmt.Errorf("error while reading log file %q: %w", logFilePath, err)
}
logrus.Infof("Logs:\n%s", string(contentBytes))

return nil
}
4 changes: 2 additions & 2 deletions internal/filesystem/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,11 @@ type IFileSystem interface {
CreateProjectInput(workspaceId, projectId string, projInput types.ProjectInput, file io.Reader, isCommon bool) error
ReadProjectInput(workspaceId, projectId, projInputId string, isCommon bool) (projInput types.ProjectInput, file io.Reader, err error)
DeleteProjectInput(workspaceId, projectId, projInputId string, isCommon bool) error
StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error
StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool, dumpCliLogs bool) error
ReadPlan(workspaceId, projectId string) (plan io.Reader, err error)
UpdatePlan(workspaceId, projectId string, plan io.Reader) error
DeletePlan(workspaceId, projectId string) error
StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error
StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool, dumpCliLogs bool) error
ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error
ReadProjectOutput(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error)
ReadProjectOutputGraph(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error)
Expand Down
2 changes: 2 additions & 0 deletions internal/move2kubeapi/handlers/handlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import (
)

const (
// DUMP_CLI_LOGS_STDOUT_PARAM is the name of the query parameter used for dump the content of CLI log file into stdout
DUMP_CLI_LOGS_STDOUT_PARAM = "dump-cli-logs"
// SKIP_QA_QUERY_PARAM is the name of the query parameter used for skipping QA
SKIP_QA_QUERY_PARAM = "skip-qa"
// REMOTE_SOURCE_QUERY_PARAM is the URL of the git remote to be used as source
Expand Down
3 changes: 2 additions & 1 deletion internal/move2kubeapi/handlers/outputs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) {
}
debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true"
skipQA := r.URL.Query().Get(SKIP_QA_QUERY_PARAM) == "true"
dumpCliLogs := r.URL.Query().Get(DUMP_CLI_LOGS_STDOUT_PARAM) == "true"
timestamp, _, err := common.GetTimestamp()
if err != nil {
logrus.Errorf("failed to get the timestamp. Error: %q", err)
Expand All @@ -66,7 +67,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) {
projOutput.Timestamp = timestamp
projOutput.Name = projOutput.Id // This isn't really used anywhere
projOutput.Status = types.ProjectOutputStatusInProgress
if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode, skipQA); err != nil {
if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode, skipQA, dumpCliLogs); err != nil {
logrus.Errorf("failed to start the transformation. Error: %q", err)
if notExErr, ok := err.(types.ErrorDoesNotExist); ok {
if notExErr.Id == "plan" {
Expand Down
3 changes: 2 additions & 1 deletion internal/move2kubeapi/handlers/plan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ func HandleStartPlanning(w http.ResponseWriter, r *http.Request) {
return
}
debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true"
if err := m2kFS.StartPlanning(workspaceId, projectId, remoteSource, debugMode); err != nil {
dumpCliLogs := r.URL.Query().Get(DUMP_CLI_LOGS_STDOUT_PARAM) == "true"
if err := m2kFS.StartPlanning(workspaceId, projectId, remoteSource, debugMode, dumpCliLogs); err != nil {
logrus.Errorf("failed to start plan generation. Error: %q", err)
if _, ok := err.(types.ErrorDoesNotExist); ok {
w.WriteHeader(http.StatusNotFound)
Expand Down

0 comments on commit 29a1df5

Please sign in to comment.