From c9ff0045405f48009680bfaf549f72dda667f58d Mon Sep 17 00:00:00 2001 From: Kin NG <59541661+k1nho@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:24:56 -0400 Subject: [PATCH] feat: flag to consume yaml file at path for configs and LRU cache never evict (#28) --- go.mod | 3 ++- go.sum | 1 + main.go | 28 +++++++++++++++++++- pkg/cache/gitrepofilepath_test.go | 13 +++++++--- pkg/cache/lrucache.go | 33 +++++++++++++++++------- pkg/cache/lrucache_test.go | 43 +++++++++++++++++++++++++------ pkg/providers/cache.go | 8 ++++-- pkg/server/server.go | 6 +++++ 8 files changed, 110 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 61c3a5a..2ac5293 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 go.uber.org/zap v1.24.0 + golang.org/x/sys v0.5.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -28,6 +30,5 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index c79ced4..c6820d7 100644 --- a/go.sum +++ b/go.sum @@ -149,4 +149,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/main.go b/main.go index 4aed885..e7d3a74 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "github.com/joho/godotenv" "go.uber.org/zap" + "gopkg.in/yaml.v3" "github.com/open-sauced/pizza/oven/pkg/database" "github.com/open-sauced/pizza/oven/pkg/providers" @@ -19,6 +20,8 @@ func main() { var err error // Initialize & parse flags + var configPath string + flag.StringVar(&configPath, "config", "", "path to .yaml file config") debugMode := flag.Bool("debug", false, "run in debug mode") flag.Parse() @@ -59,6 +62,29 @@ func main() { // Initialize the database handler pizzaOven := database.NewPizzaOvenDbHandler(databaseHost, databasePort, databaseUser, databasePwd, databaseDbName) + // Initializes configuration using a provided yaml file + config := &server.Config{NeverEvictRepos: make(map[string]bool)} + var configParser struct { + NeverEvictRepos []string `yaml:"never-evict-repos"` + } + + if configPath != "" { + configFile, err := os.ReadFile(configPath) + if err != nil { + sugarLogger.Fatalf("Could not read yaml configuration file: %s", err.Error()) + } + + err = yaml.Unmarshal(configFile, &configParser) + if err != nil { + sugarLogger.Fatalf("Could not unmarshal configuration file: %s", err.Error()) + } + + for _, repo := range configParser.NeverEvictRepos { + config.NeverEvictRepos[repo] = true + } + sugarLogger.Infof("Configuration for server was set using yaml file") + } + var pizzaGitProvider providers.GitRepoProvider switch gitProvider { case "cache": @@ -77,7 +103,7 @@ func main() { sugarLogger.Fatalf(": %s", err.Error()) } - pizzaGitProvider, err = providers.NewLRUCacheGitRepoProvider(cacheDir, minFreeDiskUint64, sugarLogger) + pizzaGitProvider, err = providers.NewLRUCacheGitRepoProvider(cacheDir, minFreeDiskUint64, sugarLogger, config.NeverEvictRepos) if err != nil { sugarLogger.Fatalf("Could not create a cache git provider: %s", err.Error()) } diff --git a/pkg/cache/gitrepofilepath_test.go b/pkg/cache/gitrepofilepath_test.go index d5a297f..3fd9d02 100644 --- a/pkg/cache/gitrepofilepath_test.go +++ b/pkg/cache/gitrepofilepath_test.go @@ -4,9 +4,10 @@ import "testing" func TestOpenAndFetch(t *testing.T) { tests := []struct { - name string - cacheDir string - repos []string + name string + cacheDir string + repos []string + neverEvictRepos map[string]bool }{ { name: "Puts repos into cache in sequential order", @@ -14,13 +15,17 @@ func TestOpenAndFetch(t *testing.T) { repos: []string{ "https://github.com/open-sauced/pizza", }, + neverEvictRepos: map[string]bool{ + "https://github.com/kubernetes/kubernetes": true, + "https://github.com/open-sauced/pizza-cli": true, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Create a new LRU cache - c, err := NewGitRepoLRUCache(tt.cacheDir, 1) + c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos) if err != nil { t.Fatalf("unexpected err: %s", err.Error()) } diff --git a/pkg/cache/lrucache.go b/pkg/cache/lrucache.go index 1dc11a2..fdf0999 100644 --- a/pkg/cache/lrucache.go +++ b/pkg/cache/lrucache.go @@ -9,6 +9,7 @@ import ( "syscall" "github.com/go-git/go-git/v5" + "golang.org/x/sys/unix" ) @@ -46,11 +47,14 @@ type GitRepoLRUCache struct { // hm is the hashmap to support the LRU cache behavior hm map[string]*list.Element + + // neverEvictRepos are the repositories that must never be evicted from the LRU cache + neverEvictRepos map[string]bool } // NewGitRepoLRUCache returns a new NewGitRepoLRUCache configured with the // destination directory to cache git repos and minimum free gbs -func NewGitRepoLRUCache(dir string, minFreeGbs uint64) (*GitRepoLRUCache, error) { +func NewGitRepoLRUCache(dir string, minFreeGbs uint64, neverEvictRepos map[string]bool) (*GitRepoLRUCache, error) { path := filepath.Clean(dir) _, err := os.Stat(path) if err != nil { @@ -72,10 +76,11 @@ func NewGitRepoLRUCache(dir string, minFreeGbs uint64) (*GitRepoLRUCache, error) } return &GitRepoLRUCache{ - minFreeDiskGb: minFreeGbs, - dir: path, - dll: list.New(), - hm: make(map[string]*list.Element), + minFreeDiskGb: minFreeGbs, + dir: path, + dll: list.New(), + hm: make(map[string]*list.Element), + neverEvictRepos: neverEvictRepos, }, nil } @@ -214,13 +219,23 @@ func (c *GitRepoLRUCache) tryEvict() error { break } + // Check that the LRU element is not part of neverEvictRepos + // If it is, find the nearest LRU not part of neverEvictRepos + lruNode := c.dll.Back() + for _, isInNeverEvictRepos := c.neverEvictRepos[lruNode.Value.(*GitRepoFilePath).key]; isInNeverEvictRepos; { + lruNode = lruNode.Next() + if lruNode == nil { + return fmt.Errorf("Disk space completely occupied by neverEvictRepos, could not evict") + } + } + // Attempt to unlock the individual element - c.dll.Back().Value.(*GitRepoFilePath).lock.Lock() + lruNode.Value.(*GitRepoFilePath).lock.Lock() // Evict least recently used repos - os.RemoveAll(c.dll.Back().Value.(*GitRepoFilePath).path) - delete(c.hm, c.dll.Back().Value.(*GitRepoFilePath).key) - c.dll.Remove(c.dll.Back()) + os.RemoveAll(lruNode.Value.(*GitRepoFilePath).path) + delete(c.hm, lruNode.Value.(*GitRepoFilePath).key) + c.dll.Remove(lruNode) // Recalculate the free bytes err = unix.Statfs(c.dir, &stat) diff --git a/pkg/cache/lrucache_test.go b/pkg/cache/lrucache_test.go index 877a7fa..5889ded 100644 --- a/pkg/cache/lrucache_test.go +++ b/pkg/cache/lrucache_test.go @@ -46,25 +46,32 @@ func TestNewGitRepoLRUCache(t *testing.T) { t.Parallel() tests := []struct { - name string - cacheDir string - wantErr bool + name string + cacheDir string + wantErr bool + neverEvictRepos map[string]bool }{ { name: "Default case", cacheDir: t.TempDir(), wantErr: false, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, { name: "Fails when directory doesn't exist", cacheDir: "/should/not/exist", wantErr: true, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, err := NewGitRepoLRUCache(tt.cacheDir, 1) + c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos) if tt.wantErr && err != nil { return } @@ -100,6 +107,7 @@ func TestPutGitRepoLRUCache(t *testing.T) { cacheDir string repos []string expectedCacheOrdering []string + neverEvictRepos map[string]bool }{ { name: "Puts repos into cache in sequential order", @@ -114,6 +122,9 @@ func TestPutGitRepoLRUCache(t *testing.T) { "https://github.com/open-sauced/pizza-cli", "https://github.com/open-sauced/pizza", }, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, { name: "Most recently used is first in order", @@ -130,12 +141,15 @@ func TestPutGitRepoLRUCache(t *testing.T) { "https://github.com/open-sauced/insights", "https://github.com/open-sauced/pizza-cli", }, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, err := NewGitRepoLRUCache(tt.cacheDir, 1) + c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos) if err != nil { t.Fatalf("unexpected err: %s", err.Error()) } @@ -161,6 +175,7 @@ func TestTryEvict(t *testing.T) { cacheDir string repos []string expectedCacheOrdering []string + neverEvictRepos map[string]bool }{ { name: "Evicts repos when size limit reached", @@ -171,12 +186,13 @@ func TestTryEvict(t *testing.T) { "https://github.com/open-sauced/insights", }, expectedCacheOrdering: []string{}, + neverEvictRepos: map[string]bool{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, err := NewGitRepoLRUCache(tt.cacheDir, 1) + c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos) if err != nil { t.Fatalf("unexpected err: %s", err.Error()) } @@ -213,6 +229,7 @@ func TestGetGitRepoLRUCache(t *testing.T) { expectedCacheOrdering []string wantErr bool wantNil bool + neverEvictRepos map[string]bool }{ { name: "Gets queried repo and inserts it to front of cache", @@ -232,6 +249,9 @@ func TestGetGitRepoLRUCache(t *testing.T) { }, wantErr: false, wantNil: false, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, { name: "Returns nothing and no error if repo not in cache", @@ -251,12 +271,15 @@ func TestGetGitRepoLRUCache(t *testing.T) { }, wantErr: false, wantNil: true, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, err := NewGitRepoLRUCache(tt.cacheDir, 1) + c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos) if err != nil { t.Fatalf("unexpected err creating cache: %s", err.Error()) } @@ -300,6 +323,7 @@ func TestGetAndPutConcurrently(t *testing.T) { expectedCacheOrdering []string wantErr bool wantNil bool + neverEvictRepos map[string]bool }{ { name: "Gets queried repo and puts it to front of cache", @@ -321,12 +345,15 @@ func TestGetAndPutConcurrently(t *testing.T) { }, wantErr: false, wantNil: false, + neverEvictRepos: map[string]bool{ + "no-test": true, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c, err := NewGitRepoLRUCache(tt.cacheDir, 1) + c, err := NewGitRepoLRUCache(tt.cacheDir, 1, tt.neverEvictRepos) if err != nil { t.Fatalf("unexpected err creating cache: %s", err.Error()) } diff --git a/pkg/providers/cache.go b/pkg/providers/cache.go index e8354b9..bd648e0 100644 --- a/pkg/providers/cache.go +++ b/pkg/providers/cache.go @@ -9,6 +9,10 @@ import ( "github.com/open-sauced/pizza/oven/pkg/cache" ) +// NeverEvictRepos holds all the repos that must never be evicted in the LRU cache +// where the key is the URL of the repo +type NeverEvictRepos map[string]bool + // LRUCacheGitRepoProvider is a git repository provider that uses an internal // Least Recently Used cache on disk for loading and querying git repositories. // LRUCacheGitRepoProvider implements and statisfies the GitRepoProvider @@ -21,8 +25,8 @@ type LRUCacheGitRepoProvider struct { // NewLRUCacheGitRepoProvider returns a new LRUCacheGitRepoProvider using the // configured cache directory and sets the minimum amount of free disk for the // cache to keep. -func NewLRUCacheGitRepoProvider(cacheDir string, minFreeDisk uint64, l *zap.SugaredLogger) (GitRepoProvider, error) { - cache, err := cache.NewGitRepoLRUCache(cacheDir, minFreeDisk) +func NewLRUCacheGitRepoProvider(cacheDir string, minFreeDisk uint64, l *zap.SugaredLogger, neverEvictRepos NeverEvictRepos) (GitRepoProvider, error) { + cache, err := cache.NewGitRepoLRUCache(cacheDir, minFreeDisk, neverEvictRepos) if err != nil { return nil, fmt.Errorf("could not initialize a new LRU cache: %s", err.Error()) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 2011b46..9021314 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -19,6 +19,12 @@ import ( "github.com/open-sauced/pizza/oven/pkg/providers" ) +// Config provides the configuration set on server startup +// - Never Evict Repos: Repos that are preserved in cache regardless of LRU policy +type Config struct { + NeverEvictRepos providers.NeverEvictRepos +} + // PizzaOvenServer provides a leveled logger for use during serving requests // and a PizzaOvenDbHanlder for accessing a sql pool of connections. type PizzaOvenServer struct {