Skip to content

Commit

Permalink
Add basic stats to Para init output
Browse files Browse the repository at this point in the history
  • Loading branch information
@ashald
ashald committed Jun 14, 2019
1 parent c10d4ba commit b51c47a
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 88 deletions.
57 changes: 38 additions & 19 deletions app/exec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"os/exec"
"os/signal"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
Expand All @@ -32,9 +33,10 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu
var stat os.FileInfo
var err error

fmt.Printf("Para is being initialized:\n")
fmt.Printf("Para is being initialized...\n")

// Plugin Dir
fmt.Printf("- Plugin Dir: ")
for _, pluginDir = range pluginDirCandidates {
expandedPath, err := homedir.Expand(pluginDir)
if err != nil {
Expand All @@ -48,7 +50,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu

if mountpoint == nil {
fmt.Printf(
" Para is humble but it won't let itself be ignored! Please make sure that at least one of the "+
"\n* Para is humble but it won't let itself be ignored! Please make sure that at least one of the "+
"following dirs exists: %s.\n",
strings.Join(pluginDirCandidates, ", "),
)
Expand All @@ -57,7 +59,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu

if !stat.IsDir() {
fmt.Printf(
" * Error: the '%s' path exists but does not appear to be a directory - please see "+
"\n* Error: the '%s' path exists but does not appear to be a directory - please see "+
"https://www.terraform.io/docs/extend/how-terraform-works.html#plugin-locations\n",
*mountpoint,
)
Expand All @@ -66,20 +68,20 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu
// Check if plugin dir is in use
pidBytes, err := ioutil.ReadFile(filepath.Join(*mountpoint, FileMeta))
if os.IsNotExist(err) {
fmt.Printf(" - Plugin Dir: %s\n", pluginDir)
fmt.Println(pluginDir)
} else {
if err != nil {
fmt.Printf(
" * Error: the previous instance of Para failed and left '%s' in a bad shape - please "+
"\n* Error: the previous instance of Para failed and left '%s' in a bad shape - please "+
"run following command to clean up: para -u %s\n", pluginDir, pluginDir,
)
} else {
fmt.Printf(" * Error: another instance of Para (PID: %s) uses '%s' right now - "+
fmt.Printf("\n* Error: another instance of Para (PID: %s) uses '%s' right now - "+
"please wait until it will finish.\n", strings.TrimSpace(string(pidBytes)), pluginDir,
)
if pluginDir == pathPluginDirUser {
fmt.Printf(
" If the other instance is running from another Terraform configuration - please "+
" If the other instance is running from another Terraform configuration - please "+
"consider creating '%s' within Terraform configuration dir to avoid contention over '%s'.\n",
pathPluginDirLocal, pathPluginDirUser,
)
Expand All @@ -89,35 +91,52 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu
}

// Cache Dir
fmt.Printf("- Cache Dir: ")
cacheDir, err := discoverCacheDir(customCachePath)
if err != nil {
fmt.Printf(
" * Error: Para requires a writable cache dir for operation but failed discovering one: %s\n",
"\n* Error: Para requires a writable cache dir for operation but failed discovering one: %s\n",
err,
)
os.Exit(1)
}
fmt.Printf(" - Cache Dir: %s\n", simplifyPath(cacheDir))
fmt.Println(simplifyPath(cacheDir))

// Primary Index
kindNameIndex, location, err := index.DiscoverIndex(primaryIndexCandidates, cacheDir, refresh)
fmt.Printf("- Primary Index: ")
loadingIndex, err := index.DiscoverIndex(primaryIndexCandidates, cacheDir, refresh)
if err != nil {
fmt.Printf(" * Error: cannnot decode primary index at '%s' as a valid YAML map: %s\n", location, err)
fmt.Printf("\n* Error: cannnot decode primary index as a valid YAML map: %s\n", err)
os.Exit(1)
}
fmt.Printf(" - Primary Index: %s\n", location)
var indexStats []string
for kind, nameToPlugins := range loadingIndex.KindToNameToPlugins {
indexStats = append(indexStats, fmt.Sprintf("%s: %d", kind, len(nameToPlugins)))
}
sort.Strings(indexStats)
fmt.Printf(
"%s as of %s (%s)\n",
loadingIndex.Location,
loadingIndex.Timestamp.Format(time.RFC3339),
strings.Join(indexStats, ", "),
)

// Index Extensions
fmt.Printf(" - Index Extensions:\n")
loadedExtensions, failedExtensions := loadExtensions(kindNameIndex, indexExtensions, refresh)
fmt.Printf("- Index Extensions: ")
loadedExtensions, failedExtensions := loadExtensions(loadingIndex, indexExtensions)
var extensionsStats []string
for _, ext := range indexExtensions {
countLoaded := loadedExtensions[ext]
countFailed := failedExtensions[ext]
fmt.Printf(" %s: loaded %d, errors %d\n", ext, countLoaded, countFailed)
extensionsStats = append(
extensionsStats,
fmt.Sprintf("%s (%d/%d)", ext, countLoaded, countLoaded+countFailed),
)
}
fmt.Printf("%s\n", strings.Join(extensionsStats, ", "))

// Command
fmt.Printf(" - Command: %s\n", strings.Join(args, " "))
fmt.Printf("- Command: %s\n", strings.Join(args, " "))

// Footer
fmt.Println()
Expand All @@ -138,7 +157,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu
}
}()
//
ready, err := mountPluginsDir(kindNameIndex.BuildPlatformIndex(), *mountpoint)
ready, err := mountPluginsDir(loadingIndex.BuildRuntimeIndex(), *mountpoint)
if err != nil {
fmt.Printf("* Para was unable to mount plugin FS over '%s': %s", pluginDir, err)
os.Exit(1)
Expand Down Expand Up @@ -167,7 +186,7 @@ func Execute(args []string, primaryIndexCandidates, indexExtensions []string, cu
}
}

func loadExtensions(index *index.LoadingIndex, extensions []string, refresh time.Duration) (loaded map[string]uint64, failed map[string]uint64) {
func loadExtensions(index *index.LoadingIndex, extensions []string) (loaded map[string]uint64, failed map[string]uint64) {
loaded = make(map[string]uint64)
failed = make(map[string]uint64)

Expand All @@ -184,7 +203,7 @@ func loadExtensions(index *index.LoadingIndex, extensions []string, refresh time
continue
}

err := index.LoadExtension(filepath.Join(expandedPath, ext.Name()), refresh)
err := index.LoadExtension(filepath.Join(expandedPath, ext.Name()))
if err != nil {
failed[path] += 1
} else {
Expand Down
148 changes: 80 additions & 68 deletions app/index/loading.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,16 @@ const fieldSize = "size"
const fieldDigest = "digest"

type LoadingIndex struct {
cacheDir string
kindToNameToPlugins map[string]map[string][]*Plugin
CacheDir string
KindToNameToPlugins map[string]map[string][]*Plugin
Timestamp time.Time
Refresh time.Duration
Location string
}

func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration) (*LoadingIndex, string, error) {
func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration) (*LoadingIndex, error) {
var content []byte
var timestamp time.Time
var location string
var err error

Expand All @@ -32,86 +36,131 @@ func DiscoverIndex(candidates []string, cacheDir string, refresh time.Duration)
continue
}

content, err = openIndex(location, cacheDir, refresh)
content, timestamp, err = openIndex(location, cacheDir, refresh)
if err != nil {
continue
}
break
}

if content == nil {
return nil, location, fmt.Errorf(
return nil, fmt.Errorf(
"failed to discover an index file at any of given locations: %s",
strings.Join(candidates, ", "),
)
}

index, err := newLoadingIndex(content, cacheDir, refresh)
return index, location, err
index := &LoadingIndex{
CacheDir: cacheDir,
KindToNameToPlugins: make(map[string]map[string][]*Plugin),
Timestamp: timestamp,
Refresh: refresh,
Location: location,
}

return index, index.loadPrimaryIndex(content)
}

func openIndex(location string, cacheDir string, refresh time.Duration) ([]byte, error) {
func openIndex(location string, cacheDir string, refresh time.Duration) ([]byte, time.Time, error) {
if xio.IsRemote(location) {
indexCacheDir := filepath.Join(cacheDir, "index")
indexCachePath := filepath.Join(indexCacheDir, crypto.DefaultStringHash(location))

cacheData, errCacheData := ioutil.ReadFile(indexCachePath)
cacheMeta, errCacheMeta := os.Stat(indexCachePath)
var cacheTimestamp time.Time
if errCacheMeta != nil {
cacheTimestamp = time.Unix(0, 0)
} else {
cacheTimestamp = cacheMeta.ModTime()
}

maxAge := time.Duration(refresh) * time.Minute
if errCacheMeta != nil || cacheMeta.ModTime().Before(time.Now().Add(-maxAge)) || errCacheData != nil {
if cacheTimestamp.Before(time.Now().Add(-maxAge)) || errCacheData != nil {
freshData, errFreshData := xio.UrlReadAll(location)
if errFreshData != nil {
if errCacheData != nil {
return nil, errFreshData // we're not sure our cache is valid and failed to fetch so just fail
return nil, time.Now(), errFreshData // we're not sure our cache is valid and failed to fetch so just fail
}
// failed to fetch but there is _some_ kind of a cache - just return it
return cacheData, nil
return cacheData, cacheTimestamp, nil
}
// we fetched fresh data - let's try to cache it but don't sweat if fail
_ = os.MkdirAll(indexCacheDir, 0755)
_ = ioutil.WriteFile(indexCachePath, freshData, 0644)
return freshData, nil
return freshData, time.Now(), nil
} else {
return cacheData, nil
return cacheData, cacheTimestamp, nil
}
} else {
return xio.UrlReadAll(location)
data, err := xio.UrlReadAll(location)
return data, time.Now(), err
}
}

func newLoadingIndex(raw []byte, cacheDir string, refresh time.Duration) (*LoadingIndex, error) {
func (i *LoadingIndex) loadPrimaryIndex(raw []byte) error {
var parsed map[string]interface{}

err := yml.Unmarshal(raw, &parsed)
if err != nil {
return nil, err
return err
}

kindToNameToPlugins := make(map[string]map[string][]*Plugin)

for kind, kindSpec := range parsed {
kindToNameToPlugins[kind] = make(map[string][]*Plugin)
i.KindToNameToPlugins[kind] = make(map[string][]*Plugin)

kindMap, kindSpecIsOk := kindSpec.(map[string]interface{})
if !kindSpecIsOk {
continue
}

for name, versionsSpec := range kindMap {
plugins := parseVersions(kind, name, versionsSpec, cacheDir, refresh)
kindToNameToPlugins[kind][name] = append(kindToNameToPlugins[kind][name], plugins...)
plugins := i.parseVersions(kind, name, versionsSpec)
i.KindToNameToPlugins[kind][name] = append(i.KindToNameToPlugins[kind][name], plugins...)
}
}

// TODO: verify collisions in a case-insensitive way
return &LoadingIndex{
cacheDir: cacheDir,
kindToNameToPlugins: kindToNameToPlugins,
}, nil
return nil
}

func parseVersions(kind, name string, versions interface{}, cacheDir string, refresh time.Duration) (result []*Plugin) {
func (i *LoadingIndex) LoadExtension(path string) error {
filename := filepath.Base(path)
if strings.ToLower(filename) != filename {
return fmt.Errorf("extension file must be in lowercase: '%s'", filename)
}
tokens := strings.SplitN(filename, ".", 3)
if len(tokens) != 3 || tokens[2] != "yaml" {
return fmt.Errorf(
"extension file name '%s' does not match expected pattern of <kind>.<name>.yaml",
filename,
)
}
kind := tokens[0]
name := tokens[1]

var versionsSpec interface{}

content, err := ioutil.ReadFile(path)
if err != nil {
return err
}
err = yml.Unmarshal(content, &versionsSpec)
if err != nil {
return err
}

plugins := i.parseVersions(kind, name, versionsSpec)

if _, ok := i.KindToNameToPlugins[kind]; !ok {
i.KindToNameToPlugins[kind] = make(map[string][]*Plugin)
}

i.KindToNameToPlugins[kind][name] = plugins

return nil
}

func (i *LoadingIndex) parseVersions(kind, name string, versions interface{}) (result []*Plugin) {
var versionMap map[string]interface{}

versionMap, versionSpecIsOk := versions.(map[string]interface{})
Expand All @@ -120,7 +169,7 @@ func parseVersions(kind, name string, versions interface{}, cacheDir string, ref
if !versionIndexUrlOk {
return
}
versionSpecBytes, err := openIndex(versionIndexUrl, cacheDir, refresh)
versionSpecBytes, _, err := openIndex(versionIndexUrl, i.CacheDir, i.Refresh)
if err != nil {
return
}
Expand Down Expand Up @@ -176,47 +225,10 @@ func parseVersions(kind, name string, versions interface{}, cacheDir string, ref
return
}

func (i *LoadingIndex) LoadExtension(path string, refresh time.Duration) error {
filename := filepath.Base(path)
if strings.ToLower(filename) != filename {
return fmt.Errorf("extension file must be in lowercase: '%s'", filename)
}
tokens := strings.SplitN(filename, ".", 3)
if len(tokens) != 3 || tokens[2] != "yaml" {
return fmt.Errorf(
"extension file name '%s' does not match expected pattern of <kind>.<name>.yaml",
filename,
)
}
kind := tokens[0]
name := tokens[1]

var versionsSpec interface{}

content, err := ioutil.ReadFile(path)
if err != nil {
return err
}
err = yml.Unmarshal(content, &versionsSpec)
if err != nil {
return err
}

plugins := parseVersions(kind, name, versionsSpec, i.cacheDir, refresh)

if _, ok := i.kindToNameToPlugins[kind]; !ok {
i.kindToNameToPlugins[kind] = make(map[string][]*Plugin)
}

i.kindToNameToPlugins[kind][name] = plugins

return nil
}

func (i *LoadingIndex) BuildPlatformIndex() *RuntimeIndex {
func (i *LoadingIndex) BuildRuntimeIndex() *RuntimeIndex {
platformToPlugins := make(map[string]map[string]*Plugin)

for _, nameToPlugins := range i.kindToNameToPlugins {
for _, nameToPlugins := range i.KindToNameToPlugins {
for _, plugins := range nameToPlugins {
for _, p := range plugins {
if _, ok := platformToPlugins[p.Platform]; !ok {
Expand All @@ -229,7 +241,7 @@ func (i *LoadingIndex) BuildPlatformIndex() *RuntimeIndex {

return &RuntimeIndex{
platformToFilenameToPlugin: platformToPlugins,
cacheDir: i.cacheDir,
cacheDir: i.CacheDir,
openFiles: make(map[string]*os.File),
alreadyOpened: make(map[string]bool),
}
Expand Down
Loading

0 comments on commit b51c47a

Please sign in to comment.