Move common error logs to debug & providing capability to provide cus…

…tom expressions (#61)

docs/docs/advanced/remote-manifest.md

@@ -28,7 +28,7 @@ harness-upgrade --secret-scope account expressions
 
 To provide custom expressions or override default expressions
 ```shell
-harness-upgrade expressions --override /path/to/file.yaml
+harness-upgrade --override /path/to/file.yaml expressions 
 ```
 
 The above command expects a yaml file whose contents are simple key values of first-gen expressions & string to replace that expression with. A sample override file would look like this -

entity.go

@@ -10,12 +10,18 @@ import (
 	"time"
 )
 
+var skipLogs = []string{
+	"already exists in the parent folder",
+	"Duplicate identifier, please try again with a new identifier",
+	"already exists in the account",
+	"already exists in this scope",
+}
+
 func CreateEntities(body RequestBody) {
 	reqId, err := QueueCreateEntity(body)
 	if err != nil {
 		return
 	}
-	log.Debugf("The request ID is %s", reqId)
 	PollForCompletion(reqId)
 }
 
@@ -40,7 +46,7 @@ func PollForCompletion(reqId string) {
 	s.Suffix = " Processing"
 	s.Start()
 	for {
-		time.Sleep(time.Second)
+		time.Sleep(time.Second * 10)
 		url := GetUrlWithQueryParams(migrationReq.Environment, MigratorService, "save/async-result", map[string]string{
 			AccountIdentifier: migrationReq.Account,
 			"requestId":       reqId,
@@ -113,7 +119,15 @@ func renderSaveSummary(saveSummary SaveSummary) {
 		log.Info("Here are the errors while migrating - ")
 		for i := range saveSummary.Errors {
 			e := saveSummary.Errors[i]
-			logWithDetails(log.ErrorLevel, e.Entity, e.Message)
+			level := log.ErrorLevel
+			// log as debug if the error is in skipLogs
+			for _, v := range skipLogs {
+				if strings.Contains(e.Message, v) {
+					level = log.DebugLevel
+					break
+				}
+			}
+			logWithDetails(level, e.Entity, e.Message)
 		}
 	}
 
@@ -144,6 +158,6 @@ func logWithDetails(level log.Level, entity CurrentGenEntity, message string) {
 			"name":  entity.Name,
 		}).Log(level, message)
 	} else {
-		log.Error(message)
+		log.WithFields(log.Fields{}).Log(level, message)
 	}
 }

expressions.go

@@ -5,7 +5,6 @@ import (
 	"github.com/jedib0t/go-pretty/v6/text"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v3"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -347,21 +346,8 @@ func getDynamicExpressionKey(key string) string {
 }
 
 func loadYamlFromFile(filePath string) {
-	filePath = strings.TrimSpace(filePath)
-	if len(filePath) == 0 {
-		return
-	}
-	yFile, err := os.ReadFile(filePath)
-	if err != nil {
-		log.Fatal(err)
-	}
-	data := make(map[string]string)
-	err = yaml.Unmarshal(yFile, &data)
-	if err != nil {
-		log.Fatal(err)
-	}
+	data := LoadYamlFromFile(filePath)
 	for k, v := range data {
 		ExpressionsMap[k] = v
 	}
-	log.Infof("Successfully loaded %d custom expressions from the file", len(data))
 }

helper.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"os"
 	"strings"
 
@@ -338,3 +339,21 @@ func MigrateEntities(promptConfirm bool, scopes []string, pluralValue string, en
 
 	return nil
 }
+
+func LoadYamlFromFile(filePath string) map[string]string {
+	filePath = strings.TrimSpace(filePath)
+	if len(filePath) == 0 {
+		return nil
+	}
+	yFile, err := os.ReadFile(filePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	data := make(map[string]string)
+	err = yaml.Unmarshal(yFile, &data)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Infof("Successfully loaded %d custom expressions from the file", len(data))
+	return data
+}

main.go

@@ -57,6 +57,7 @@ var migrationReq = struct {
 
 func getReqBody(entityType EntityType, filter Filter) RequestBody {
 	inputs := Inputs{
+		Expressions: LoadYamlFromFile(migrationReq.CustomExpressionsFile),
 		Defaults: Defaults{
 			Secret:                EntityDefaults{Scope: getOrDefault(migrationReq.SecretScope, Project)},
 			SecretManager:         EntityDefaults{Scope: getOrDefault(migrationReq.SecretScope, Project)},
@@ -220,6 +221,11 @@ func main() {
 			Usage:       "`API_KEY` for the target account to authenticate & authorise the migration.",
 			Destination: &migrationReq.TargetAuthToken,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "override",
+			Usage:       "provide a `FILE` to load custom expressions from",
+			Destination: &migrationReq.CustomExpressionsFile,
+		}),
 	}
 	app := &cli.App{
 		Name:                 "harness-upgrade",
@@ -476,11 +482,6 @@ func main() {
 						DefaultText: "json,yaml,yml",
 						Destination: &migrationReq.FileExtensions,
 					},
-					&cli.StringFlag{
-						Name:        "override",
-						Usage:       "provide a `FILE` to load custom expressions from",
-						Destination: &migrationReq.CustomExpressionsFile,
-					},
 				},
 				Action: func(context *cli.Context) error {
 					return cliWrapper(ReplaceCurrentGenExpressionsWithNextGen, context)

types.go

@@ -39,7 +39,8 @@ type Defaults struct {
 }
 
 type Inputs struct {
-	Defaults Defaults `json:"defaults"`
+	Expressions map[string]string `json:"expressions"`
+	Defaults    Defaults          `json:"defaults"`
 }
 
 type OrgDetails struct {