Skip to content

Commit

Permalink
Merge pull request #310 from akgalwas/add-scripts-for-shoot-comparison-1
Browse files Browse the repository at this point in the history
Compare directories in the shoot_comparator tool
  • Loading branch information
kyma-bot authored Aug 6, 2024
2 parents d6e2ae7 + 18a8726 commit 4c950b8
Show file tree
Hide file tree
Showing 9 changed files with 670 additions and 48 deletions.
85 changes: 84 additions & 1 deletion hack/shoot-comparator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,87 @@ CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o ./bin/shoot-comparator ./cmd
To compare files, execute the following command:
```
shoot-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM>
```
```

To compare directories, execute the following command:
```
shoot-comparator dirs <directory with shoot files generated by Provisioner> <directory with shoot files generated by KIM>
```

In order to save details of the comparison such as the exact differences between the files, use the `--outputDir` flag:
```
shoot-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM> --outputDir <output directory>
```

In order to specify the date from which the shoots should be compared, use the `--fromDate` flag:
```
shoot-comparator files <shoot file generated by Provisioner> <shoot file generated by KIM> --fromDate <date>
```

## Interpretation of the results

### Compared directories are equal
The following is an example of the `shoot-comparator` output when no differences are found:
```
2024/07/30 09:00:18 INFO Comparing directories: /tmp/kim-test/shoot-comparator/test1/kim and /tmp/kim-test/shoot-comparator/test1/provisioner
2024/07/30 09:00:18 INFO Saving comparison details
2024/07/30 09:00:18 INFO Results stored in "/tmp/kim-test/shoot-comparator/test1/results/2024-07-30T09:00:18+02:00"
2024/07/30 09:00:18 INFO No differences found.
```
The output directory will contain the `result.txt` file. Exemplary content of the file:
```
Comparing files older than:0001-01-01 00:00:00 +0000 UTC
Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test1/kim directory = 2
Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test1/provisioner directory = 2
No differences found.
```

### Compared directories differ
The following is an example of the `shoot-comparator` output when differences are found:
```
2024/07/30 08:56:01 INFO Comparing directories: /tmp/kim-test/shoot-comparator/test2/kim and /tmp/kim-test/shoot-comparator/test2/provisioner
2024/07/30 08:56:01 INFO Saving comparison details
2024/07/30 08:56:01 INFO Results stored in "/tmp/kim-test/shoot-comparator/test2/results/2024-07-30T08:56:01+02:00"
2024/07/30 08:56:01 WARN Differences found.
```

Exemplary content of the `result.txt` file:
```
Comparing files older than:0001-01-01 00:00:00 +0000 UTC
Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test2/kim directory = 2
Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test2/provisioner directory = 2
Differences found.
------------------------------------------------------------------------------------------
Files that differ:
- shoot1.yaml
------------------------------------------------------------------------------------------
```

The output directory will also contain the `shoot1.yaml.diff` file with detailed file comparison results.

The following is an example of the `result.txt` file when some files exist only in one of the directories:
```
Comparing files older than:0001-01-01 00:00:00 +0000 UTC
Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/kim directory = 22
Number of files in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/provisioner directory = 157
Differences found.
------------------------------------------------------------------------------------------
Files existing in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/kim folder only:
- garden-kyma-dev-kim-pvc-test3.yaml
- garden-kyma-dev-kims-aws-pvc-shootCR.yaml
------------------------------------------------------------------------------------------
------------------------------------------------------------------------------------------
Files existing in /Users/i326211/dev/temp/kim-test/shoot-comparator/test3/provisioner folder only:
- garden-kyma-dev-a2f8da9.yaml
- garden-kyma-dev-a3008c5.yaml
------------------------------------------------------------------------------------------
```
95 changes: 71 additions & 24 deletions hack/shoot-comparator/cmd/comparator/directories.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,45 +4,92 @@ import (
"fmt"
"github.com/kyma-project/infrastructure-manager/tools/shoot-comparator/internal/directories"
"github.com/spf13/cobra"
"log/slog"
"time"
)

func init() {
rootCmd.AddCommand(directoriesCmd)
directoriesCmd.Flags().String("fromDate", "", "Files older than specified date will not be compared")
directoriesCmd.Flags().String("outputDir", "", "Directory storing comparison results")
}

var directoriesCmd = &cobra.Command{
Use: "dirs",
Aliases: []string{"d"},
Short: "Compare directories",
Short: "Compares directories",
Args: cobra.ExactArgs(2),
Long: "Compares two directories and checks if they are equal. " +
"By default, all files are compared. " +
"Use the --fromDate flag to specify the date from which the comparison should be performed. " +
"Use the --outputDir flag to save the comparison results.",
Run: func(cmd *cobra.Command, args []string) {
leftDir := args[0]
rightDir := args[1]

fmt.Printf("Comparing directories: %s and %s \n", leftDir, rightDir)
result, err := directories.CompareDirectories(leftDir, rightDir)
fromDateString, err := cmd.Flags().GetString("fromDate")
if err != nil {
fmt.Printf("Failed to compare directories: %s \n", err.Error())
fmt.Printf("Error occurred when parsing command line arguments:%q", err)
return
}
fmt.Printf("Numer of files in %s directory = %d \n", leftDir, result.LeftDirFilesCount)
fmt.Printf("Numer of files in %s directory = %d \n", rightDir, result.RightDirFilesCount)

if result.Equal {
fmt.Println("Directories are equal")
} else {
fmt.Println("Directories are NOT equal")
if len(result.LeftOnly) != 0 {
fmt.Printf("Files existing in %s folder only: %s \n", leftDir, result.LeftOnly)
}

if len(result.RightOnly) != 0 {
fmt.Printf("Files existing in %s folder only: %s \n", rightDir, result.RightOnly)
}

if len(result.Diff) != 0 {
fmt.Printf("Differences found: %s \n", result.Diff)
}

fromDate, err := parseFromDate(fromDateString)
if err != nil {
slog.Error(fmt.Sprintf("Error occurred when parsing command line arguments: %q", err))
return
}

outputDir, err := cmd.Flags().GetString("outputDir")
if err != nil {
slog.Error(fmt.Sprintf("Error occurred when parsing command line arguments: %q", err))
return
}

if !fromDate.IsZero() {
slog.Info(fmt.Sprintf("Only files created after the following date: %v will be compared.", fromDate))
}

slog.Info(fmt.Sprintf("Comparing directories: %s and %s", leftDir, rightDir))
result, err := directories.CompareDirectories(leftDir, rightDir, time.Time{})
if err != nil {
slog.Error(fmt.Sprintf("Failed to compare directories: %q", err.Error()))
return
}

if outputDir != "" {
writeReport(outputDir, result, fromDate)
}
logComparisonResults(result)
},
}

func init() {
rootCmd.AddCommand(directoriesCmd)
func writeReport(outputDir string, result directories.Result, fromDate time.Time) {
slog.Info("Saving comparison details")
resultsDir, err := directories.SaveComparisonReport(result, outputDir, fromDate)
if err != nil {
fmt.Printf("Failed to save comparison report: %s \n", err.Error())
} else {
slog.Info(fmt.Sprintf("Results stored in %q", resultsDir))
}
}

func parseFromDate(fromDateString string) (time.Time, error) {
if fromDateString != "" {
fromDate, err := time.Parse(time.RFC3339, fromDateString)
if err != nil {
return time.Time{}, fmt.Errorf("failed to parse `fromDate': %q", err)
}

return fromDate, nil
}

return time.Time{}, nil
}

func logComparisonResults(comparisonResult directories.Result) {
if comparisonResult.Equal {
slog.Info("No differences found.")
} else {
slog.Warn("Differences found.")
}
}
5 changes: 4 additions & 1 deletion hack/shoot-comparator/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ require (
github.com/onsi/ginkgo/v2 v2.19.0
github.com/onsi/gomega v1.33.1
github.com/spf13/cobra v1.8.1
k8s.io/apimachinery v0.29.6
github.com/stretchr/testify v1.9.0
sigs.k8s.io/yaml v1.4.0
)

require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand All @@ -22,6 +23,7 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.21.0 // indirect
Expand All @@ -32,6 +34,7 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.29.6 // indirect
k8s.io/apiextensions-apiserver v0.29.6 // indirect
k8s.io/apimachinery v0.29.6 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
Expand Down
4 changes: 2 additions & 2 deletions hack/shoot-comparator/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down
138 changes: 138 additions & 0 deletions hack/shoot-comparator/internal/directories/comparator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package directories

import (
"github.com/kyma-project/infrastructure-manager/tools/shoot-comparator/internal/files"
"os"
"path"
"slices"
"time"
)

type Result struct {
Equal bool
LeftDir string
RightDir string
LeftOnly []string
RightOnly []string
LeftDirFilesCount int
RightDirFilesCount int
Diff []Difference
}

type Difference struct {
Filename string
LeftFile string
RightFile string
Message string
}

func CompareDirectories(leftDir, rightDir string, olderThan time.Time) (Result, error) {

leftFileNames, err := getFileNames(leftDir, olderThan)
if err != nil {
return Result{}, err
}

rightFileNames, err := getFileNames(rightDir, olderThan)
if err != nil {
return Result{}, err
}

fileNamesToCompare := getIntersection(leftFileNames, rightFileNames)
differences, err := compare(fileNamesToCompare, leftDir, rightDir)

filesExistInLeftDirOnly := filterOut(leftFileNames, fileNamesToCompare)
filesExistInRightDirOnly := filterOut(rightFileNames, fileNamesToCompare)

equal := len(differences) == 0 && len(filesExistInLeftDirOnly) == 0 && len(filesExistInRightDirOnly) == 0

return Result{
LeftDir: leftDir,
RightDir: rightDir,
Equal: equal,
Diff: differences,
RightOnly: filesExistInRightDirOnly,
LeftOnly: filesExistInLeftDirOnly,
LeftDirFilesCount: len(leftFileNames),
RightDirFilesCount: len(rightFileNames),
}, nil
}

func getFileNames(dir string, olderThan time.Time) ([]string, error) {
dirEntries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}

var fileNames []string

for _, dirEntry := range dirEntries {
if dirEntry.IsDir() {
continue
}

fileInfo, err := dirEntry.Info()
if err != nil {
return nil, err
}

if fileInfo.ModTime().After(olderThan) {
fileNames = append(fileNames, dirEntry.Name())
}
}

slices.Sort(fileNames)

return fileNames, nil
}

func getIntersection(leftFiles []string, rightFiles []string) []string {
var intersection []string
for _, leftFile := range leftFiles {
_, found := slices.BinarySearch(rightFiles, leftFile)
if found {
intersection = append(intersection, leftFile)
}
}

return intersection
}

func filterOut(fullFileList []string, filesToFilterOut []string) []string {
var result []string
for _, file := range fullFileList {
_, found := slices.BinarySearch(filesToFilterOut, file)

if !found {
result = append(result, file)
}
}

return result
}

func compare(filesNames []string, leftDir, rightDir string) ([]Difference, error) {
var differences []Difference

for _, fileName := range filesNames {
leftFilePath := path.Join(leftDir, fileName)
rightFilePath := path.Join(rightDir, fileName)

equal, diffMessage, err := files.CompareFiles(leftFilePath, rightFilePath)
if err != nil {
return nil, err
}

if !equal {
diff := Difference{
Filename: fileName,
LeftFile: leftFilePath,
RightFile: rightFilePath,
Message: diffMessage,
}
differences = append(differences, diff)
}
}

return differences, nil
}
Loading

0 comments on commit 4c950b8

Please sign in to comment.