Compare directories in the shoot_comparator tool

hack/shoot-comparator/README.md

@@ -16,5 +16,88 @@ CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o ./bin/comparator ./cmd
 
 To compare files, execute the following command:
 ```
-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM>
-```
+shoot-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM>
+```
+
+To compare directories, execute the following command:
+```
+shoot-comparator dirs <directory with shoot files generated by Provisioner> <directory with shoot files generated by KIM>
+```
+
+In order to save details of the comparison such as the exact differences between the files, use the `--outputDir` flag:
+```
+shoot-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM> --outputDir <output directory>
+```
+
+In order to specify the date from which the shoots should be compared, use the `--fromDate` flag:
+```
+shoot-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM> --fromDate <date>
+``` 
+
+## Interpretation of the results 
+
+### Compared directories are equal 
+The following is an example of the `shoot-comparator` output when no differences are found:
+```
+2024/07/30 09:00:18 INFO Comparing directories: /tmp/kim-test/shoot-comparator/test1/kim and /tmp/kim-test/shoot-comparator/test1/provisioner
+2024/07/30 09:00:18 INFO Saving comparison details
+2024/07/30 09:00:18 INFO Results stored in "/tmp/kim-test/shoot-comparator/test1/results/2024-07-30T09:00:18+02:00"
+2024/07/30 09:00:18 INFO No differences found.
+```
+The output directory will contain the `result.txt` file. Exemplary content of the file:
+```
+Comparing files older than:0001-01-01 00:00:00 +0000 UTC
+
+Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test1/kim directory = 2
+Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test1/provisioner directory = 2
+
+No differences found.
+```
+
+### Compared directories differ
+The following is an example of the `shoot-comparator` output when differences are found:
+```
+2024/07/30 08:56:01 INFO Comparing directories: /tmp/kim-test/shoot-comparator/test2/kim and /tmp/kim-test/shoot-comparator/test2/provisioner
+2024/07/30 08:56:01 INFO Saving comparison details
+2024/07/30 08:56:01 INFO Results stored in "/tmp/kim-test/shoot-comparator/test2/results/2024-07-30T08:56:01+02:00"
+2024/07/30 08:56:01 WARN Differences found.
+```
+
+Exemplary content of the `result.txt` file:
+```
+Comparing files older than:0001-01-01 00:00:00 +0000 UTC
+
+Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test2/kim directory = 2
+Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test2/provisioner directory = 2
+
+Differences found.
+
+------------------------------------------------------------------------------------------
+Files that differ: 
+- shoot1.yaml
+------------------------------------------------------------------------------------------
+```
+
+The output directory will also contain the `shoot1.yaml.diff` file with detailed file comparison results. 
+
+The following is an example of the `result.txt` file when some files exist only in one of the directories:
+```
+Comparing files older than:0001-01-01 00:00:00 +0000 UTC
+
+Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/kim directory = 22
+Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/provisioner directory = 157
+
+Differences found.
+
+Files existing in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/kim folder only:
+------------------------------------------------------------------------------------------
+garden-kyma-dev-kim-pvc-test3.yaml
+garden-kyma-dev-kims-aws-pvc-shootCR.yaml
+------------------------------------------------------------------------------------------
+
+Files existing in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/provisioner folder only:
+------------------------------------------------------------------------------------------
+garden-kyma-dev-a2f8da9.yaml
+garden-kyma-dev-a3008c5.yaml
+------------------------------------------------------------------------------------------
+```

hack/shoot-comparator/cmd/comparator/directories.go

@@ -4,45 +4,92 @@ import (
 	"fmt"
 	"github.com/kyma-project/infrastructure-manager/tools/shoot-comparator/internal/directories"
 	"github.com/spf13/cobra"
+	"log/slog"
+	"time"
 )
 
+func init() {
+	rootCmd.AddCommand(directoriesCmd)
+	directoriesCmd.Flags().String("fromDate", "", "Files older than specified date will not be compared")
+	directoriesCmd.Flags().String("outputDir", "", "Directory storing comparison results")
+}
+
 var directoriesCmd = &cobra.Command{
 	Use:     "dirs",
 	Aliases: []string{"d"},
-	Short:   "Compare directories",
+	Short:   "Compares directories",
 	Args:    cobra.ExactArgs(2),
+	Long: "Compares two directories and checks if they are equal. " +
+		"By default, all files are compared. " +
+		"Use the --fromDate flag to specify the date from which the comparison should be performed. " +
+		"Use the --outputDir flag to save the comparison results.",
 	Run: func(cmd *cobra.Command, args []string) {
 		leftDir := args[0]
 		rightDir := args[1]
 
-		fmt.Printf("Comparing directories: %s and %s \n", leftDir, rightDir)
-		result, err := directories.CompareDirectories(leftDir, rightDir)
+		fromDateString, err := cmd.Flags().GetString("fromDate")
 		if err != nil {
-			fmt.Printf("Failed to compare directories: %s \n", err.Error())
+			fmt.Printf("Error occurred when parsing command line arguments:%q", err)
 			return
 		}
-		fmt.Printf("Numer of files in %s directory = %d \n", leftDir, result.LeftDirFilesCount)
-		fmt.Printf("Numer of files in %s directory = %d \n", rightDir, result.RightDirFilesCount)
-
-		if result.Equal {
-			fmt.Println("Directories are equal")
-		} else {
-			fmt.Println("Directories are NOT equal")
-			if len(result.LeftOnly) != 0 {
-				fmt.Printf("Files existing in %s folder only: %s \n", leftDir, result.LeftOnly)
-			}
-
-			if len(result.RightOnly) != 0 {
-				fmt.Printf("Files existing in %s folder only: %s \n", rightDir, result.RightOnly)
-			}
-
-			if len(result.Diff) != 0 {
-				fmt.Printf("Differences found: %s \n", result.Diff)
-			}
+
+		fromDate, err := parseFromDate(fromDateString)
+		if err != nil {
+			slog.Error(fmt.Sprintf("Error occurred when parsing command line arguments: %q", err))
+			return
 		}
+
+		outputDir, err := cmd.Flags().GetString("outputDir")
+		if err != nil {
+			slog.Error(fmt.Sprintf("Error occurred when parsing command line arguments: %q", err))
+			return
+		}
+
+		if !fromDate.IsZero() {
+			slog.Info(fmt.Sprintf("Only files created after the following date: %v will be compared.", fromDate))
+		}
+
+		slog.Info(fmt.Sprintf("Comparing directories: %s and %s", leftDir, rightDir))
+		result, err := directories.CompareDirectories(leftDir, rightDir, time.Time{})
+		if err != nil {
+			slog.Error(fmt.Sprintf("Failed to compare directories: %q", err.Error()))
+			return
+		}
+
+		if outputDir != "" {
+			writeReport(outputDir, result, fromDate)
+		}
+		logComparisonResults(result)
 	},
 }
 
-func init() {
-	rootCmd.AddCommand(directoriesCmd)
+func writeReport(outputDir string, result directories.Result, fromDate time.Time) {
+	slog.Info("Saving comparison details")
+	resultsDir, err := directories.SaveComparisonReport(result, outputDir, fromDate)
+	if err != nil {
+		fmt.Printf("Failed to save comparison report: %s \n", err.Error())
+	} else {
+		slog.Info(fmt.Sprintf("Results stored in %q", resultsDir))
+	}
+}
+
+func parseFromDate(fromDateString string) (time.Time, error) {
+	if fromDateString != "" {
+		fromDate, err := time.Parse(time.RFC3339, fromDateString)
+		if err != nil {
+			return time.Time{}, fmt.Errorf("failed to parse `fromDate': %q", err)
+		}
+
+		return fromDate, nil
+	}
+
+	return time.Time{}, nil
+}
+
+func logComparisonResults(comparisonResult directories.Result) {
+	if comparisonResult.Equal {
+		slog.Info("No differences found.")
+	} else {
+		slog.Warn("Differences found.")
+	}
 }

hack/shoot-comparator/go.mod

@@ -7,11 +7,12 @@ require (
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	github.com/spf13/cobra v1.8.1
-	k8s.io/apimachinery v0.29.6
+	github.com/stretchr/testify v1.9.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -22,6 +23,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
@@ -32,6 +34,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.29.6 // indirect
 	k8s.io/apiextensions-apiserver v0.29.6 // indirect
+	k8s.io/apimachinery v0.29.6 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

hack/shoot-comparator/go.sum

@@ -76,8 +76,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

hack/shoot-comparator/internal/directories/comparator.go

@@ -0,0 +1,138 @@
+package directories
+
+import (
+	"github.com/kyma-project/infrastructure-manager/tools/shoot-comparator/internal/files"
+	"os"
+	"path"
+	"slices"
+	"time"
+)
+
+type Result struct {
+	Equal              bool
+	LeftDir            string
+	RightDir           string
+	LeftOnly           []string
+	RightOnly          []string
+	LeftDirFilesCount  int
+	RightDirFilesCount int
+	Diff               []Difference
+}
+
+type Difference struct {
+	Filename  string
+	LeftFile  string
+	RightFile string
+	Message   string
+}
+
+func CompareDirectories(leftDir, rightDir string, olderThan time.Time) (Result, error) {
+
+	leftFileNames, err := getFileNames(leftDir, olderThan)
+	if err != nil {
+		return Result{}, err
+	}
+
+	rightFileNames, err := getFileNames(rightDir, olderThan)
+	if err != nil {
+		return Result{}, err
+	}
+
+	fileNamesToCompare := getIntersection(leftFileNames, rightFileNames)
+	differences, err := compare(fileNamesToCompare, leftDir, rightDir)
+
+	filesExistInLeftDirOnly := filterOut(leftFileNames, fileNamesToCompare)
+	filesExistInRightDirOnly := filterOut(rightFileNames, fileNamesToCompare)
+
+	equal := len(differences) == 0 && len(filesExistInLeftDirOnly) == 0 && len(filesExistInRightDirOnly) == 0
+
+	return Result{
+		LeftDir:            leftDir,
+		RightDir:           rightDir,
+		Equal:              equal,
+		Diff:               differences,
+		RightOnly:          filesExistInRightDirOnly,
+		LeftOnly:           filesExistInLeftDirOnly,
+		LeftDirFilesCount:  len(leftFileNames),
+		RightDirFilesCount: len(rightFileNames),
+	}, nil
+}
+
+func getFileNames(dir string, olderThan time.Time) ([]string, error) {
+	dirEntries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	var fileNames []string
+
+	for _, dirEntry := range dirEntries {
+		if dirEntry.IsDir() {
+			continue
+		}
+
+		fileInfo, err := dirEntry.Info()
+		if err != nil {
+			return nil, err
+		}
+
+		if fileInfo.ModTime().After(olderThan) {
+			fileNames = append(fileNames, dirEntry.Name())
+		}
+	}
+
+	slices.Sort(fileNames)
+
+	return fileNames, nil
+}
+
+func getIntersection(leftFiles []string, rightFiles []string) []string {
+	var intersection []string
+	for _, leftFile := range leftFiles {
+		_, found := slices.BinarySearch(rightFiles, leftFile)
+		if found {
+			intersection = append(intersection, leftFile)
+		}
+	}
+
+	return intersection
+}
+
+func filterOut(fullFileList []string, filesToFilterOut []string) []string {
+	var result []string
+	for _, file := range fullFileList {
+		_, found := slices.BinarySearch(filesToFilterOut, file)
+
+		if !found {
+			result = append(result, file)
+		}
+	}
+
+	return result
+}
+
+func compare(filesNames []string, leftDir, rightDir string) ([]Difference, error) {
+	var differences []Difference
+
+	for _, fileName := range filesNames {
+		leftFilePath := path.Join(leftDir, fileName)
+		rightFilePath := path.Join(rightDir, fileName)
+
+		equal, diffMessage, err := files.CompareFiles(leftFilePath, rightFilePath)
+		if err != nil {
+			return nil, err
+		}
+
+		if !equal {
+			diff := Difference{
+				Filename:  fileName,
+				LeftFile:  leftFilePath,
+				RightFile: rightFilePath,
+				Message:   diffMessage,
+			}
+			differences = append(differences, diff)
+		}
+	}
+
+	return differences, nil
+}