From b51c47a35b48fcfa4059767f517bcf8b05904303 Mon Sep 17 00:00:00 2001 From: Borys Pierov Date: Wed, 12 Jun 2019 23:43:36 -0400 Subject: [PATCH] Add basic stats to Para init output --- app/exec.go | 57 +++++++++++------ app/index/loading.go | 148 +++++++++++++++++++++++-------------------- app/paths.go | 2 +- 3 files changed, 119 insertions(+), 88 deletions(-) diff --git a/app/exec.go b/app/exec.go index 4e79b72..2e7c44a 100644 --- a/app/exec.go +++ b/app/exec.go @@ -11,6 +11,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "sort" "strings" "syscall" "time" @@ -32,9 +33,10 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu var stat os.FileInfo var err error - fmt.Printf("Para is being initialized:\n") + fmt.Printf("Para is being initialized...\n") // Plugin Dir + fmt.Printf("- Plugin Dir: ") for _, pluginDir = range pluginDirCandidates { expandedPath, err := homedir.Expand(pluginDir) if err != nil { @@ -48,7 +50,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu if mountpoint == nil { fmt.Printf( - " Para is humble but it won't let itself be ignored! Please make sure that at least one of the "+ + "\n* Para is humble but it won't let itself be ignored! Please make sure that at least one of the "+ "following dirs exists: %s.\n", strings.Join(pluginDirCandidates, ", "), ) @@ -57,7 +59,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu if !stat.IsDir() { fmt.Printf( - " * Error: the '%s' path exists but does not appear to be a directory - please see "+ + "\n* Error: the '%s' path exists but does not appear to be a directory - please see "+ "https://www.terraform.io/docs/extend/how-terraform-works.html#plugin-locations\n", *mountpoint, ) @@ -66,20 +68,20 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu // Check if plugin dir is in use pidBytes, err := ioutil.ReadFile(filepath.Join(*mountpoint, FileMeta)) if os.IsNotExist(err) { - fmt.Printf(" - Plugin Dir: %s\n", pluginDir) + fmt.Println(pluginDir) } else { if err != nil { fmt.Printf( - " * Error: the previous instance of Para failed and left '%s' in a bad shape - please "+ + "\n* Error: the previous instance of Para failed and left '%s' in a bad shape - please "+ "run following command to clean up: para -u %s\n", pluginDir, pluginDir, ) } else { - fmt.Printf(" * Error: another instance of Para (PID: %s) uses '%s' right now - "+ + fmt.Printf("\n* Error: another instance of Para (PID: %s) uses '%s' right now - "+ "please wait until it will finish.\n", strings.TrimSpace(string(pidBytes)), pluginDir, ) if pluginDir == pathPluginDirUser { fmt.Printf( - " If the other instance is running from another Terraform configuration - please "+ + " If the other instance is running from another Terraform configuration - please "+ "consider creating '%s' within Terraform configuration dir to avoid contention over '%s'.\n", pathPluginDirLocal, pathPluginDirUser, ) @@ -89,35 +91,52 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu } // Cache Dir + fmt.Printf("- Cache Dir: ") cacheDir, err := discoverCacheDir(customCachePath) if err != nil { fmt.Printf( - " * Error: Para requires a writable cache dir for operation but failed discovering one: %s\n", + "\n* Error: Para requires a writable cache dir for operation but failed discovering one: %s\n", err, ) os.Exit(1) } - fmt.Printf(" - Cache Dir: %s\n", simplifyPath(cacheDir)) + fmt.Println(simplifyPath(cacheDir)) // Primary Index - kindNameIndex, location, err := index.DiscoverIndex(primaryIndexCandidates, cacheDir, refresh) + fmt.Printf("- Primary Index: ") + loadingIndex, err := index.DiscoverIndex(primaryIndexCandidates, cacheDir, refresh) if err != nil { - fmt.Printf(" * Error: cannnot decode primary index at '%s' as a valid YAML map: %s\n", location, err) + fmt.Printf("\n* Error: cannnot decode primary index as a valid YAML map: %s\n", err) os.Exit(1) } - fmt.Printf(" - Primary Index: %s\n", location) + var indexStats []string + for kind, nameToPlugins := range loadingIndex.KindToNameToPlugins { + indexStats = append(indexStats, fmt.Sprintf("%s: %d", kind, len(nameToPlugins))) + } + sort.Strings(indexStats) + fmt.Printf( + "%s as of %s (%s)\n", + loadingIndex.Location, + loadingIndex.Timestamp.Format(time.RFC3339), + strings.Join(indexStats, ", "), + ) // Index Extensions - fmt.Printf(" - Index Extensions:\n") - loadedExtensions, failedExtensions := loadExtensions(kindNameIndex, indexExtensions, refresh) + fmt.Printf("- Index Extensions: ") + loadedExtensions, failedExtensions := loadExtensions(loadingIndex, indexExtensions) + var extensionsStats []string for _, ext := range indexExtensions { countLoaded := loadedExtensions[ext] countFailed := failedExtensions[ext] - fmt.Printf(" %s: loaded %d, errors %d\n", ext, countLoaded, countFailed) + extensionsStats = append( + extensionsStats, + fmt.Sprintf("%s (%d/%d)", ext, countLoaded, countLoaded+countFailed), + ) } + fmt.Printf("%s\n", strings.Join(extensionsStats, ", ")) // Command - fmt.Printf(" - Command: %s\n", strings.Join(args, " ")) + fmt.Printf("- Command: %s\n", strings.Join(args, " ")) // Footer fmt.Println() @@ -138,7 +157,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu } }() // - ready, err := mountPluginsDir(kindNameIndex.BuildPlatformIndex(), *mountpoint) + ready, err := mountPluginsDir(loadingIndex.BuildRuntimeIndex(), *mountpoint) if err != nil { fmt.Printf("* Para was unable to mount plugin FS over '%s': %s", pluginDir, err) os.Exit(1) @@ -167,7 +186,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu } } -func loadExtensions(index *index.LoadingIndex, extensions []string, refresh time.Duration) (loaded map[string]uint64, failed map[string]uint64) { +func loadExtensions(index *index.LoadingIndex, extensions []string) (loaded map[string]uint64, failed map[string]uint64) { loaded = make(map[string]uint64) failed = make(map[string]uint64) @@ -184,7 +203,7 @@ func loadExtensions(index *index.LoadingIndex, extensions []string, refresh time continue } - err := index.LoadExtension(filepath.Join(expandedPath, ext.Name()), refresh) + err := index.LoadExtension(filepath.Join(expandedPath, ext.Name())) if err != nil { failed[path] += 1 } else { diff --git a/app/index/loading.go b/app/index/loading.go index e3dd578..1301508 100644 --- a/app/index/loading.go +++ b/app/index/loading.go @@ -18,12 +18,16 @@ const fieldSize = "size" const fieldDigest = "digest" type LoadingIndex struct { - cacheDir string - kindToNameToPlugins map[string]map[string][]*Plugin + CacheDir string + KindToNameToPlugins map[string]map[string][]*Plugin + Timestamp time.Time + Refresh time.Duration + Location string } -func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration) (*LoadingIndex, string, error) { +func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration) (*LoadingIndex, error) { var content []byte + var timestamp time.Time var location string var err error @@ -32,7 +36,7 @@ func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration) continue } - content, err = openIndex(location, cacheDir, refresh) + content, timestamp, err = openIndex(location, cacheDir, refresh) if err != nil { continue } @@ -40,58 +44,70 @@ func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration) } if content == nil { - return nil, location, fmt.Errorf( + return nil, fmt.Errorf( "failed to discover an index file at any of given locations: %s", strings.Join(candidates, ", "), ) } - index, err := newLoadingIndex(content, cacheDir, refresh) - return index, location, err + index := &LoadingIndex{ + CacheDir: cacheDir, + KindToNameToPlugins: make(map[string]map[string][]*Plugin), + Timestamp: timestamp, + Refresh: refresh, + Location: location, + } + + return index, index.loadPrimaryIndex(content) } -func openIndex(location string, cacheDir string, refresh time.Duration) ([]byte, error) { +func openIndex(location string, cacheDir string, refresh time.Duration) ([]byte, time.Time, error) { if xio.IsRemote(location) { indexCacheDir := filepath.Join(cacheDir, "index") indexCachePath := filepath.Join(indexCacheDir, crypto.DefaultStringHash(location)) cacheData, errCacheData := ioutil.ReadFile(indexCachePath) cacheMeta, errCacheMeta := os.Stat(indexCachePath) + var cacheTimestamp time.Time + if errCacheMeta != nil { + cacheTimestamp = time.Unix(0, 0) + } else { + cacheTimestamp = cacheMeta.ModTime() + } maxAge := time.Duration(refresh) * time.Minute - if errCacheMeta != nil || cacheMeta.ModTime().Before(time.Now().Add(-maxAge)) || errCacheData != nil { + if cacheTimestamp.Before(time.Now().Add(-maxAge)) || errCacheData != nil { freshData, errFreshData := xio.UrlReadAll(location) if errFreshData != nil { if errCacheData != nil { - return nil, errFreshData // we're not sure our cache is valid and failed to fetch so just fail + return nil, time.Now(), errFreshData // we're not sure our cache is valid and failed to fetch so just fail } // failed to fetch but there is _some_ kind of a cache - just return it - return cacheData, nil + return cacheData, cacheTimestamp, nil } // we fetched fresh data - let's try to cache it but don't sweat if fail _ = os.MkdirAll(indexCacheDir, 0755) _ = ioutil.WriteFile(indexCachePath, freshData, 0644) - return freshData, nil + return freshData, time.Now(), nil } else { - return cacheData, nil + return cacheData, cacheTimestamp, nil } } else { - return xio.UrlReadAll(location) + data, err := xio.UrlReadAll(location) + return data, time.Now(), err } } -func newLoadingIndex(raw []byte, cacheDir string, refresh time.Duration) (*LoadingIndex, error) { +func (i *LoadingIndex) loadPrimaryIndex(raw []byte) error { var parsed map[string]interface{} err := yml.Unmarshal(raw, &parsed) if err != nil { - return nil, err + return err } - kindToNameToPlugins := make(map[string]map[string][]*Plugin) - for kind, kindSpec := range parsed { - kindToNameToPlugins[kind] = make(map[string][]*Plugin) + i.KindToNameToPlugins[kind] = make(map[string][]*Plugin) kindMap, kindSpecIsOk := kindSpec.(map[string]interface{}) if !kindSpecIsOk { @@ -99,19 +115,52 @@ func newLoadingIndex(raw []byte, cacheDir string, refresh time.Duration) (*Loadi } for name, versionsSpec := range kindMap { - plugins := parseVersions(kind, name, versionsSpec, cacheDir, refresh) - kindToNameToPlugins[kind][name] = append(kindToNameToPlugins[kind][name], plugins...) + plugins := i.parseVersions(kind, name, versionsSpec) + i.KindToNameToPlugins[kind][name] = append(i.KindToNameToPlugins[kind][name], plugins...) } } - // TODO: verify collisions in a case-insensitive way - return &LoadingIndex{ - cacheDir: cacheDir, - kindToNameToPlugins: kindToNameToPlugins, - }, nil + return nil } -func parseVersions(kind, name string, versions interface{}, cacheDir string, refresh time.Duration) (result []*Plugin) { +func (i *LoadingIndex) LoadExtension(path string) error { + filename := filepath.Base(path) + if strings.ToLower(filename) != filename { + return fmt.Errorf("extension file must be in lowercase: '%s'", filename) + } + tokens := strings.SplitN(filename, ".", 3) + if len(tokens) != 3 || tokens[2] != "yaml" { + return fmt.Errorf( + "extension file name '%s' does not match expected pattern of ..yaml", + filename, + ) + } + kind := tokens[0] + name := tokens[1] + + var versionsSpec interface{} + + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + err = yml.Unmarshal(content, &versionsSpec) + if err != nil { + return err + } + + plugins := i.parseVersions(kind, name, versionsSpec) + + if _, ok := i.KindToNameToPlugins[kind]; !ok { + i.KindToNameToPlugins[kind] = make(map[string][]*Plugin) + } + + i.KindToNameToPlugins[kind][name] = plugins + + return nil +} + +func (i *LoadingIndex) parseVersions(kind, name string, versions interface{}) (result []*Plugin) { var versionMap map[string]interface{} versionMap, versionSpecIsOk := versions.(map[string]interface{}) @@ -120,7 +169,7 @@ func parseVersions(kind, name string, versions interface{}, cacheDir string, ref if !versionIndexUrlOk { return } - versionSpecBytes, err := openIndex(versionIndexUrl, cacheDir, refresh) + versionSpecBytes, _, err := openIndex(versionIndexUrl, i.CacheDir, i.Refresh) if err != nil { return } @@ -176,47 +225,10 @@ func parseVersions(kind, name string, versions interface{}, cacheDir string, ref return } -func (i *LoadingIndex) LoadExtension(path string, refresh time.Duration) error { - filename := filepath.Base(path) - if strings.ToLower(filename) != filename { - return fmt.Errorf("extension file must be in lowercase: '%s'", filename) - } - tokens := strings.SplitN(filename, ".", 3) - if len(tokens) != 3 || tokens[2] != "yaml" { - return fmt.Errorf( - "extension file name '%s' does not match expected pattern of ..yaml", - filename, - ) - } - kind := tokens[0] - name := tokens[1] - - var versionsSpec interface{} - - content, err := ioutil.ReadFile(path) - if err != nil { - return err - } - err = yml.Unmarshal(content, &versionsSpec) - if err != nil { - return err - } - - plugins := parseVersions(kind, name, versionsSpec, i.cacheDir, refresh) - - if _, ok := i.kindToNameToPlugins[kind]; !ok { - i.kindToNameToPlugins[kind] = make(map[string][]*Plugin) - } - - i.kindToNameToPlugins[kind][name] = plugins - - return nil -} - -func (i *LoadingIndex) BuildPlatformIndex() *RuntimeIndex { +func (i *LoadingIndex) BuildRuntimeIndex() *RuntimeIndex { platformToPlugins := make(map[string]map[string]*Plugin) - for _, nameToPlugins := range i.kindToNameToPlugins { + for _, nameToPlugins := range i.KindToNameToPlugins { for _, plugins := range nameToPlugins { for _, p := range plugins { if _, ok := platformToPlugins[p.Platform]; !ok { @@ -229,7 +241,7 @@ func (i *LoadingIndex) BuildPlatformIndex() *RuntimeIndex { return &RuntimeIndex{ platformToFilenameToPlugin: platformToPlugins, - cacheDir: i.cacheDir, + cacheDir: i.CacheDir, openFiles: make(map[string]*os.File), alreadyOpened: make(map[string]bool), } diff --git a/app/paths.go b/app/paths.go index 297d0e1..ecab4c8 100644 --- a/app/paths.go +++ b/app/paths.go @@ -27,4 +27,4 @@ func simplifyPath(path string) string { } } return path -} \ No newline at end of file +}