feat: flag to consume yaml file at path for configs and LRU cache nev…

…er evict (#28)

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/joho/godotenv v1.5.1
 	github.com/lib/pq v1.10.9
 	go.uber.org/zap v1.24.0
+	golang.org/x/sys v0.5.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -28,6 +30,5 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.6.0 // indirect
 	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

go.sum

@@ -149,4 +149,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

main.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/joho/godotenv"
 	"go.uber.org/zap"
+	"gopkg.in/yaml.v3"
 
 	"github.com/open-sauced/pizza/oven/pkg/database"
 	"github.com/open-sauced/pizza/oven/pkg/providers"
@@ -19,6 +20,8 @@ func main() {
 	var err error
 
 	// Initialize & parse flags
+	var configPath string
+	flag.StringVar(&configPath, "config", "", "path to .yaml file config")
 	debugMode := flag.Bool("debug", false, "run in debug mode")
 	flag.Parse()
 
@@ -59,6 +62,29 @@ func main() {
 	// Initialize the database handler
 	pizzaOven := database.NewPizzaOvenDbHandler(databaseHost, databasePort, databaseUser, databasePwd, databaseDbName)
 
+	// Initializes configuration using a provided yaml file
+	config := &server.Config{NeverEvictRepos: make(map[string]bool)}
+	var configParser struct {
+		NeverEvictRepos []string `yaml:"never-evict-repos"`
+	}
+
+	if configPath != "" {
+		configFile, err := os.ReadFile(configPath)
+		if err != nil {
+			sugarLogger.Fatalf("Could not read yaml configuration file: %s", err.Error())
+		}
+
+		err = yaml.Unmarshal(configFile, &configParser)
+		if err != nil {
+			sugarLogger.Fatalf("Could not unmarshal configuration file: %s", err.Error())
+		}
+
+		for _, repo := range configParser.NeverEvictRepos {
+			config.NeverEvictRepos[repo] = true
+		}
+		sugarLogger.Infof("Configuration for server was set using yaml file")
+	}
+
 	var pizzaGitProvider providers.GitRepoProvider
 	switch gitProvider {
 	case "cache":
@@ -77,7 +103,7 @@ func main() {
 			sugarLogger.Fatalf(": %s", err.Error())
 		}
 
-		pizzaGitProvider, err = providers.NewLRUCacheGitRepoProvider(cacheDir, minFreeDiskUint64, sugarLogger)
+		pizzaGitProvider, err = providers.NewLRUCacheGitRepoProvider(cacheDir, minFreeDiskUint64, sugarLogger, config.NeverEvictRepos)
 		if err != nil {
 			sugarLogger.Fatalf("Could not create a cache git provider: %s", err.Error())
 		}

pkg/cache/gitrepofilepath_test.go

@@ -4,23 +4,28 @@ import "testing"
 
 func TestOpenAndFetch(t *testing.T) {
 	tests := []struct {
-		name     string
-		cacheDir string
-		repos    []string
+		name            string
+		cacheDir        string
+		repos           []string
+		neverEvictRepos map[string]bool
 	}{
 		{
 			name:     "Puts repos into cache in sequential order",
 			cacheDir: t.TempDir(),
 			repos: []string{
 				"https://github.com/open-sauced/pizza",
 			},
+			neverEvictRepos: map[string]bool{
+				"https://github.com/kubernetes/kubernetes": true,
+				"https://github.com/open-sauced/pizza-cli": true,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			// Create a new LRU cache
-			c, err := NewGitRepoLRUCache(tt.cacheDir, 1)
+			c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos)
 			if err != nil {
 				t.Fatalf("unexpected err: %s", err.Error())
 			}

pkg/cache/lrucache.go

@@ -9,6 +9,7 @@ import (
 	"syscall"
 
 	"github.com/go-git/go-git/v5"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -46,11 +47,14 @@ type GitRepoLRUCache struct {
 
 	// hm is the hashmap to support the LRU cache behavior
 	hm map[string]*list.Element
+
+	// neverEvictRepos are the repositories that must never be evicted from the LRU cache
+	neverEvictRepos map[string]bool
 }
 
 // NewGitRepoLRUCache returns a new NewGitRepoLRUCache configured with the
 // destination directory to cache git repos and minimum free gbs
-func NewGitRepoLRUCache(dir string, minFreeGbs uint64) (*GitRepoLRUCache, error) {
+func NewGitRepoLRUCache(dir string, minFreeGbs uint64, neverEvictRepos map[string]bool) (*GitRepoLRUCache, error) {
 	path := filepath.Clean(dir)
 	_, err := os.Stat(path)
 	if err != nil {
@@ -72,10 +76,11 @@ func NewGitRepoLRUCache(dir string, minFreeGbs uint64) (*GitRepoLRUCache, error)
 	}
 
 	return &GitRepoLRUCache{
-		minFreeDiskGb: minFreeGbs,
-		dir:           path,
-		dll:           list.New(),
-		hm:            make(map[string]*list.Element),
+		minFreeDiskGb:   minFreeGbs,
+		dir:             path,
+		dll:             list.New(),
+		hm:              make(map[string]*list.Element),
+		neverEvictRepos: neverEvictRepos,
 	}, nil
 }
 
@@ -214,13 +219,23 @@ func (c *GitRepoLRUCache) tryEvict() error {
 			break
 		}
 
+		// Check that the LRU element is not part of neverEvictRepos
+		// If it is, find the nearest LRU not part of neverEvictRepos
+		lruNode := c.dll.Back()
+		for _, isInNeverEvictRepos := c.neverEvictRepos[lruNode.Value.(*GitRepoFilePath).key]; isInNeverEvictRepos; {
+			lruNode = lruNode.Next()
+			if lruNode == nil {
+				return fmt.Errorf("Disk space completely occupied by neverEvictRepos, could not evict")
+			}
+		}
+
 		// Attempt to unlock the individual element
-		c.dll.Back().Value.(*GitRepoFilePath).lock.Lock()
+		lruNode.Value.(*GitRepoFilePath).lock.Lock()
 
 		// Evict least recently used repos
-		os.RemoveAll(c.dll.Back().Value.(*GitRepoFilePath).path)
-		delete(c.hm, c.dll.Back().Value.(*GitRepoFilePath).key)
-		c.dll.Remove(c.dll.Back())
+		os.RemoveAll(lruNode.Value.(*GitRepoFilePath).path)
+		delete(c.hm, lruNode.Value.(*GitRepoFilePath).key)
+		c.dll.Remove(lruNode)
 
 		// Recalculate the free bytes
 		err = unix.Statfs(c.dir, &stat)

pkg/cache/lrucache_test.go

@@ -46,25 +46,32 @@ func TestNewGitRepoLRUCache(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
-		name     string
-		cacheDir string
-		wantErr  bool
+		name            string
+		cacheDir        string
+		wantErr         bool
+		neverEvictRepos map[string]bool
 	}{
 		{
 			name:     "Default case",
 			cacheDir: t.TempDir(),
 			wantErr:  false,
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 		{
 			name:     "Fails when directory doesn't exist",
 			cacheDir: "/should/not/exist",
 			wantErr:  true,
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := NewGitRepoLRUCache(tt.cacheDir, 1)
+			c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos)
 			if tt.wantErr && err != nil {
 				return
 			}
@@ -100,6 +107,7 @@ func TestPutGitRepoLRUCache(t *testing.T) {
 		cacheDir              string
 		repos                 []string
 		expectedCacheOrdering []string
+		neverEvictRepos       map[string]bool
 	}{
 		{
 			name:     "Puts repos into cache in sequential order",
@@ -114,6 +122,9 @@ func TestPutGitRepoLRUCache(t *testing.T) {
 				"https://github.com/open-sauced/pizza-cli",
 				"https://github.com/open-sauced/pizza",
 			},
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 		{
 			name:     "Most recently used is first in order",
@@ -130,12 +141,15 @@ func TestPutGitRepoLRUCache(t *testing.T) {
 				"https://github.com/open-sauced/insights",
 				"https://github.com/open-sauced/pizza-cli",
 			},
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := NewGitRepoLRUCache(tt.cacheDir, 1)
+			c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos)
 			if err != nil {
 				t.Fatalf("unexpected err: %s", err.Error())
 			}
@@ -161,6 +175,7 @@ func TestTryEvict(t *testing.T) {
 		cacheDir              string
 		repos                 []string
 		expectedCacheOrdering []string
+		neverEvictRepos       map[string]bool
 	}{
 		{
 			name:     "Evicts repos when size limit reached",
@@ -171,12 +186,13 @@ func TestTryEvict(t *testing.T) {
 				"https://github.com/open-sauced/insights",
 			},
 			expectedCacheOrdering: []string{},
+			neverEvictRepos:       map[string]bool{},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := NewGitRepoLRUCache(tt.cacheDir, 1)
+			c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos)
 			if err != nil {
 				t.Fatalf("unexpected err: %s", err.Error())
 			}
@@ -213,6 +229,7 @@ func TestGetGitRepoLRUCache(t *testing.T) {
 		expectedCacheOrdering []string
 		wantErr               bool
 		wantNil               bool
+		neverEvictRepos       map[string]bool
 	}{
 		{
 			name:     "Gets queried repo and inserts it to front of cache",
@@ -232,6 +249,9 @@ func TestGetGitRepoLRUCache(t *testing.T) {
 			},
 			wantErr: false,
 			wantNil: false,
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 		{
 			name:     "Returns nothing and no error if repo not in cache",
@@ -251,12 +271,15 @@ func TestGetGitRepoLRUCache(t *testing.T) {
 			},
 			wantErr: false,
 			wantNil: true,
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := NewGitRepoLRUCache(tt.cacheDir, 1)
+			c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos)
 			if err != nil {
 				t.Fatalf("unexpected err creating cache: %s", err.Error())
 			}
@@ -300,6 +323,7 @@ func TestGetAndPutConcurrently(t *testing.T) {
 		expectedCacheOrdering []string
 		wantErr               bool
 		wantNil               bool
+		neverEvictRepos       map[string]bool
 	}{
 		{
 			name:     "Gets queried repo and puts it to front of cache",
@@ -321,12 +345,15 @@ func TestGetAndPutConcurrently(t *testing.T) {
 			},
 			wantErr: false,
 			wantNil: false,
+			neverEvictRepos: map[string]bool{
+				"no-test": true,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c, err := NewGitRepoLRUCache(tt.cacheDir, 1)
+			c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos)
 			if err != nil {
 				t.Fatalf("unexpected err creating cache: %s", err.Error())
 			}

pkg/providers/cache.go

@@ -9,6 +9,10 @@ import (
 	"github.com/open-sauced/pizza/oven/pkg/cache"
 )
 
+// NeverEvictRepos holds all the repos that must never be evicted in the LRU cache
+// where the key is the URL of the repo
+type NeverEvictRepos map[string]bool
+
 // LRUCacheGitRepoProvider is a git repository provider that uses an internal
 // Least Recently Used cache on disk for loading and querying git repositories.
 // LRUCacheGitRepoProvider implements and statisfies the GitRepoProvider
@@ -21,8 +25,8 @@ type LRUCacheGitRepoProvider struct {
 // NewLRUCacheGitRepoProvider returns a new LRUCacheGitRepoProvider using the
 // configured cache directory and sets the minimum amount of free disk for the
 // cache to keep.
-func NewLRUCacheGitRepoProvider(cacheDir string, minFreeDisk uint64, l *zap.SugaredLogger) (GitRepoProvider, error) {
-	cache, err := cache.NewGitRepoLRUCache(cacheDir, minFreeDisk)
+func NewLRUCacheGitRepoProvider(cacheDir string, minFreeDisk uint64, l *zap.SugaredLogger, neverEvictRepos NeverEvictRepos) (GitRepoProvider, error) {
+	cache, err := cache.NewGitRepoLRUCache(cacheDir, minFreeDisk, neverEvictRepos)
 	if err != nil {
 		return nil, fmt.Errorf("could not initialize a new LRU cache: %s", err.Error())
 	}

pkg/server/server.go

@@ -19,6 +19,12 @@ import (
 	"github.com/open-sauced/pizza/oven/pkg/providers"
 )
 
+// Config provides the configuration set on server startup
+// - Never Evict Repos: Repos that are preserved in cache regardless of LRU policy
+type Config struct {
+	NeverEvictRepos providers.NeverEvictRepos
+}
+
 // PizzaOvenServer provides a leveled logger for use during serving requests
 // and a PizzaOvenDbHanlder for accessing a sql pool of connections.
 type PizzaOvenServer struct {