From e3a82fe33dd97d95d40eb30c651edcf1b8b25f44 Mon Sep 17 00:00:00 2001 From: Tony Worm Date: Sat, 11 Apr 2020 17:03:55 -0600 Subject: [PATCH] building blocks and parts of the whole --- cue.sums | 2 + docs/process.md | 35 ++++++ go.mod | 7 ++ go.sum | 14 +++ lib/cache/checksum.go | 33 ++++++ lib/cache/local.go | 43 ++++++++ lib/cache/lookup.go | 33 ++++++ lib/cache/write.go | 22 ++++ lib/hack.go | 94 ++++++++++++++++ lib/modder/modder_check.go | 29 +++++ lib/modder/modder_errors.go | 18 +++- lib/modder/modder_graph.go | 2 +- lib/modder/modder_load.go | 52 +++++++-- lib/modder/modder_resolve.go | 1 + lib/modder/modder_status.go | 5 +- lib/modder/modder_tidy.go | 2 +- lib/modder/modder_vendor.go | 8 +- lib/modder/modder_verify.go | 30 +++++- lib/modder/modder_write.go | 6 +- lib/parse/sumfile/sumfile.go | 19 +++- lib/repos/github/client.go | 27 +++++ lib/repos/github/tags.go | 66 ++++++++++++ lib/repos/tags.go | 19 ++++ lib/util/billy.go | 203 ++++++++++++++++++++++++++--------- lib/util/glob.go | 39 +++++++ 25 files changed, 734 insertions(+), 75 deletions(-) create mode 100644 cue.sums create mode 100644 docs/process.md create mode 100644 lib/cache/checksum.go create mode 100644 lib/cache/local.go create mode 100644 lib/cache/lookup.go create mode 100644 lib/cache/write.go create mode 100644 lib/modder/modder_resolve.go create mode 100644 lib/repos/github/client.go create mode 100644 lib/repos/github/tags.go create mode 100644 lib/repos/tags.go create mode 100644 lib/util/glob.go diff --git a/cue.sums b/cue.sums new file mode 100644 index 0000000..d67dbac --- /dev/null +++ b/cue.sums @@ -0,0 +1,2 @@ +github.com/hofstadter-io/hofmod-cli v0.3.0 h1:+wCqHV08voZyt87uKKFpNOst6wON+OFOEqTViIF4Enk= +github.com/hofstadter-io/hofmod-cli v0.3.0/cue.mods h1:hvuTtdDNMFf5D9Jv0SfQfg2qRBOkFGb+EVOwkVtYKQI= diff --git a/docs/process.md b/docs/process.md new file mode 100644 index 0000000..9b9ca78 --- /dev/null +++ b/docs/process.md @@ -0,0 +1,35 @@ +# MVS Processing + +This is pseudo code for how MVS handles dependencies + +From: `mvs vendor ` (when no lang, loops over discovered) + +### 1. Read Root Module + +1. Read MVS / mod files + 1. `.mvsconfig` if exists (this will configure paths and behaviors for this module) + 1. `.mod` file, EXIT if does not exist, warn to init + 1. `.sum` file, if exists +1. If compare(sum, mod) == Same / Valid + 1. Check content is ok + 1. If not, EXIT and WARN +1. For each entry in {mod - sum} + 1. Fetch: cache -> remote + 1. CALL: ReadDepsModule + + + + + +### FN: ReadDepsModule + + +1. Read MVS / mod files + 1. `.mvsconfig` if exists (this will configure paths and behaviors for this module) + 1. `.mod` file, EXIT if does not exist, warn to init + 1. `.sum` file, if exists +1. + + + + diff --git a/go.mod b/go.mod index eb736d1..df3dc04 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,15 @@ require ( github.com/bmatcuk/doublestar v1.2.2 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 + github.com/google/go-github/v30 v30.1.0 + github.com/parnurzeal/gorequest v0.2.16 github.com/spf13/cobra v0.0.6 github.com/spf13/viper v1.4.0 // indirect golang.org/x/mod v0.2.0 + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + moul.io/http2curl v1.0.0 // indirect ) + +// replace github.com/hofstadter-io/go-utils => ../go-utils diff --git a/go.sum b/go.sum index d102cec..96c1ada 100644 --- a/go.sum +++ b/go.sum @@ -60,10 +60,18 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= +github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -101,6 +109,8 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuH github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ= +github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -172,6 +182,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -199,6 +210,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -220,3 +232,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= diff --git a/lib/cache/checksum.go b/lib/cache/checksum.go new file mode 100644 index 0000000..2bc7d32 --- /dev/null +++ b/lib/cache/checksum.go @@ -0,0 +1,33 @@ +package cache + +import ( + "fmt" + "path/filepath" + "os" + + "golang.org/x/mod/sumdb/dirhash" +) + +func Checksum(lang, mod, ver string) (string, error) { + + dir := filepath.Join( + LocalCacheBaseDir, + "mod", + lang, + mod, + "@", + ver, + ) + + fmt.Println("Cache Load:", dir) + + _, err := os.Lstat(dir) + if err != nil { + return "", err + } + + h, err := dirhash.HashDir(dir, mod, dirhash.Hash1) + + return h, err +} + diff --git a/lib/cache/local.go b/lib/cache/local.go new file mode 100644 index 0000000..dff1d76 --- /dev/null +++ b/lib/cache/local.go @@ -0,0 +1,43 @@ +package cache + +import ( + "path/filepath" + + "github.com/hofstadter-io/mvs/lib/util" +) + +var ( + LocalCacheBaseDir = filepath.Join(util.UserHomeDir(), ".mvs") +) + +func SetBaseDir(basedir string) { + LocalCacheBaseDir = basedir +} + +/* +func Lookup(modFile, mdr, mod, ver string) (zdata []byte, err error) { + + dir := filepath.Join( + LocalCacheBaseDir, + "mod", + mdr, + mod, + "@", + ver, + ) + + fmt.Println("Cache Lookup:", dir) + + // TODO, lookup + + var buf bytes.Buffer + + m := module.Version{ Path: mod, Version: ver } + err = zip.CreateFromDir(&buf, m, dir, modFile, []string{"**.*"}, nil) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} +*/ diff --git a/lib/cache/lookup.go b/lib/cache/lookup.go new file mode 100644 index 0000000..5acdff2 --- /dev/null +++ b/lib/cache/lookup.go @@ -0,0 +1,33 @@ +package cache + +import ( + "fmt" + "path/filepath" + "os" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/osfs" +) + +func Load(lang, mod, ver string) (FS billy.Filesystem, err error) { + + dir := filepath.Join( + LocalCacheBaseDir, + "mod", + lang, + mod, + "@", + ver, + ) + + fmt.Println("Cache Load:", dir) + + _, err = os.Lstat(dir) + if err != nil { + return nil, err + } + + FS = osfs.New(dir) + + return FS, nil +} diff --git a/lib/cache/write.go b/lib/cache/write.go new file mode 100644 index 0000000..477fe82 --- /dev/null +++ b/lib/cache/write.go @@ -0,0 +1,22 @@ +package cache + +import ( + "path/filepath" + + "github.com/go-git/go-billy/v5" + + "github.com/hofstadter-io/mvs/lib/util" +) + +func Write(lang, remote, owner, repo, tag string, FS billy.Filesystem) error { + outdir := filepath.Join( + LocalCacheBaseDir, + "mod", + lang, + remote, + owner, + repo + "@" + tag, + ) + + return util.BillyWriteDirToOS(outdir, "/", FS) +} diff --git a/lib/hack.go b/lib/hack.go index 741e272..bfb5b00 100644 --- a/lib/hack.go +++ b/lib/hack.go @@ -2,10 +2,104 @@ package lib import ( "fmt" + "io/ioutil" + "strings" + + googithub "github.com/google/go-github/v30/github" + "github.com/go-git/go-billy/v5/memfs" + + "github.com/hofstadter-io/mvs/lib/cache" + "github.com/hofstadter-io/mvs/lib/parse/sumfile" + "github.com/hofstadter-io/mvs/lib/repos/github" + "github.com/hofstadter-io/mvs/lib/util" ) func Hack(lang string, args []string) error { fmt.Println("Hack", args) + client, err := github.NewClient() + if err != nil { + return err + } + + owner := args[0] + repo := args[1] + tag := args[2] + + tags, err := github.GetTags(client, owner, repo) + if err != nil { + return err + } + + // The tag we are looking for + var T *googithub.RepositoryTag + for _, t := range tags { + if tag != "" && tag == *t.Name { + T = t + fmt.Printf("FOUND ") + } + fmt.Println(*t.Name, *t.Commit.SHA) + } + + // Fetch and write to cache if tag found + if T != nil { + zReader, err := github.FetchTagZip(client, T) + if err != nil { + return fmt.Errorf("While fetching zipfile\n%w\n", err) + } + FS := memfs.New() + + err = util.BillyLoadFromZip(zReader, FS, true) + if err != nil { + return fmt.Errorf("While reading zipfile\n%w\n", err) + } + + // fmt.Println("GOT HERE 1") + + err = cache.Write("hof", "github.com", owner, repo, tag, FS) + if err != nil { + return fmt.Errorf("While writing to cache\n%w\n", err) + } + + // fmt.Println("GOT HERE 2") + + dirhash, err := util.BillyCalcHash(FS) + if err != nil { + return fmt.Errorf("While calculating dir hash\n%w\n", err) + } + + modhash, err := util.BillyCalcFileHash("cue.mods", FS) + if err != nil { + return fmt.Errorf("While calculating mod hash\n%w\n", err) + } + + S := sumfile.Sum{ + Mods: make(map[sumfile.Version][]string), + } + + dver := sumfile.Version{ + Path: strings.Join([]string{"github.com", owner, repo}, "/"), + Version: tag, + } + S.Add(dver, dirhash) + + mver := sumfile.Version{ + Path: strings.Join([]string{"github.com", owner, repo}, "/"), + Version: strings.Join([]string{tag, "cue.mods"}, "/"), + } + S.Add(mver, modhash) + + fmt.Println("=====") + out, err := S.Write() + if err != nil { + return err + } + fmt.Println(out) + fmt.Println("=====") + + ioutil.WriteFile("cue.sums", []byte(out), 0644) + + } + return nil } diff --git a/lib/modder/modder_check.go b/lib/modder/modder_check.go index da5c09b..bc329eb 100644 --- a/lib/modder/modder_check.go +++ b/lib/modder/modder_check.go @@ -55,6 +55,8 @@ func (mdr *Modder) CheckAndFetchRootDeps() error { continue } + // TODO Lookup in cache + // HANDLE remote and non-local replace the same way ref, refs, err := git.IndexGitRemote(R.NewPath, R.NewVersion) if err != nil { @@ -251,6 +253,33 @@ func (mdr *Modder) CompareModToSum() error { return nil } +func (mdr *Modder) FindPresentMissingInSum() ([]string, []string, error) { + present := []string{} + missing := []string{} + + mod := mdr.module + sf := mod.SumFile + if sf == nil { + return nil, nil, fmt.Errorf("No sum file %q for %s, run 'mvs vendor [%s]' to fix.", mdr.SumFile, mdr.Name, mdr.Name) + } + + for path, R := range mod.SelfDeps { + ver := sumfile.Version{ + Path: path, + Version: R.NewVersion, + } + + _, ok := sf.Mods[ver] + if ok { + present = append(present, path) + } else { + missing = append(missing, path) + } + } + + return present, missing, nil +} + func (mdr *Modder) CompareSumToVendor() error { return nil diff --git a/lib/modder/modder_errors.go b/lib/modder/modder_errors.go index 06c2ecc..d20b0f0 100644 --- a/lib/modder/modder_errors.go +++ b/lib/modder/modder_errors.go @@ -5,13 +5,16 @@ import ( ) func (mdr *Modder) CheckForErrors() error { + if len(mdr.errors) > 0 { + return fmt.Errorf("Exiting due to errors in modder %s", mdr.Name) + } if len(mdr.module.Errors) > 0 { - return fmt.Errorf("Exiting due to errors during vendoring in %s.", mdr.module.Module) + return fmt.Errorf("Exiting due to errors in module %s", mdr.module.Module) } for _, dep := range mdr.depsMap { if len(dep.Errors) > 0 { - return fmt.Errorf("Exiting due to errors during vendoring in %s.", dep.Module) + return fmt.Errorf("Exiting due to errors in module %s", dep.Module) } } @@ -21,8 +24,15 @@ func (mdr *Modder) CheckForErrors() error { func (mdr *Modder) PrintErrors() error { var wasError error + if len(mdr.errors) > 0 { + wasError = fmt.Errorf("Exiting due to errors.") + for _, err := range mdr.errors { + fmt.Println(err) + } + } + if len(mdr.module.Errors) > 0 { - wasError = fmt.Errorf("Exiting due to errors during vendoring.") + wasError = fmt.Errorf("Exiting due to errors.") for _, err := range mdr.module.Errors { fmt.Println(err) } @@ -31,7 +41,7 @@ func (mdr *Modder) PrintErrors() error { for _, dep := range mdr.depsMap { if len(dep.Errors) > 0 { if wasError != nil { - wasError = fmt.Errorf("Exiting due to errors during vendoring.") + wasError = fmt.Errorf("Exiting due to errors.") } for _, err := range dep.Errors { fmt.Println(err) diff --git a/lib/modder/modder_graph.go b/lib/modder/modder_graph.go index fc8f9f0..ded1eb7 100644 --- a/lib/modder/modder_graph.go +++ b/lib/modder/modder_graph.go @@ -33,7 +33,7 @@ func (mdr *Modder) Graph() error { func (mdr *Modder) GraphMVS() error { // Load minimal root module - err := mdr.LoadMinimalFromFS(".") + err := mdr.LoadMetaFromFS(".") if err != nil { return err } diff --git a/lib/modder/modder_load.go b/lib/modder/modder_load.go index c58b893..e5d420a 100644 --- a/lib/modder/modder_load.go +++ b/lib/modder/modder_load.go @@ -10,9 +10,20 @@ import ( - MappingFile */ -func (mdr *Modder) LoadMinimalFromFS(dir string) error { +/* +func (mdr *Modder) LoadRootFromFS() error { // Load the root module - err := mdr.LoadRootFromFS(".") + err := mdr.LoadMetaFromFS(".") + if err != nil { + return err + } + + return nil +} + +func (mdr *Modder) LoadModderFromFS(dir string) error { + // Load the root module + err := mdr.LoadMetaFromFS(dir) if err != nil { return err } @@ -22,7 +33,7 @@ func (mdr *Modder) LoadMinimalFromFS(dir string) error { func (mdr *Modder) LoadIndexDepsFromFS(dir string) error { // Load the root module - err := mdr.LoadRootFromFS(".") + err := mdr.LoadMetaFromFS(".") if err != nil { return err } @@ -36,8 +47,29 @@ func (mdr *Modder) LoadIndexDepsFromFS(dir string) error { return nil } +*/ + +func (mdr *Modder) LoadMetaFromBilly() error { + // Load the root module + err := mdr.LoadMetaFromFS(".") + if err != nil { + return err + } + + return nil +} + +func (mdr *Modder) LoadMetaFromZip() error { + // Load the root module + err := mdr.LoadMetaFromFS(".") + if err != nil { + return err + } + + return nil +} -func (mdr *Modder) LoadRootFromFS(dir string) error { +func (mdr *Modder) LoadMetaFromFS(dir string) error { // Shortcut for no load modules, forget the reason for no load... if mdr.NoLoad { return nil @@ -54,17 +86,17 @@ func (mdr *Modder) LoadRootFromFS(dir string) error { // Load module files var err error - err = mdr.LoadRootModFile() + err = mdr.LoadModFile() if err != nil { return err } - err = mdr.LoadRootSumFile() + err = mdr.LoadSumFile() if err != nil { return err } - err = mdr.LoadRootMappingsFile() + err = mdr.LoadMappingsFile() if err != nil { return err } @@ -73,7 +105,7 @@ func (mdr *Modder) LoadRootFromFS(dir string) error { } // Loads the root modules mod file -func (mdr *Modder) LoadRootModFile() error { +func (mdr *Modder) LoadModFile() error { fn := mdr.ModFile m := mdr.module @@ -88,7 +120,7 @@ func (mdr *Modder) LoadRootModFile() error { } // Loads the root modules sum file -func (mdr *Modder) LoadRootSumFile() error { +func (mdr *Modder) LoadSumFile() error { fn := mdr.SumFile m := mdr.module @@ -101,7 +133,7 @@ func (mdr *Modder) LoadRootSumFile() error { } // Loads the root modules mapping file -func (mdr *Modder) LoadRootMappingsFile() error { +func (mdr *Modder) LoadMappingsFile() error { fn := mdr.MappingFile m := mdr.module diff --git a/lib/modder/modder_resolve.go b/lib/modder/modder_resolve.go new file mode 100644 index 0000000..45cca36 --- /dev/null +++ b/lib/modder/modder_resolve.go @@ -0,0 +1 @@ +package modder diff --git a/lib/modder/modder_status.go b/lib/modder/modder_status.go index 87858fe..c8e6c5b 100644 --- a/lib/modder/modder_status.go +++ b/lib/modder/modder_status.go @@ -34,7 +34,7 @@ func (mdr *Modder) StatusMVS() error { var err error // Load minimal root module - err = mdr.LoadMinimalFromFS(".") + err = mdr.LoadMetaFromFS(".") if err != nil { return err } @@ -52,10 +52,11 @@ func (mdr *Modder) StatusMVS() error { fmt.Println("==================") if sf != nil { - err = sf.Print() + out, err := sf.Write() if err != nil { return err } + fmt.Println(out) } else { fmt.Printf("No sum file %q found for lang %q\n", mdr.SumFile, mdr.Name) diff --git a/lib/modder/modder_tidy.go b/lib/modder/modder_tidy.go index a735f54..b47b2f2 100644 --- a/lib/modder/modder_tidy.go +++ b/lib/modder/modder_tidy.go @@ -33,7 +33,7 @@ func (mdr *Modder) Tidy() error { func (mdr *Modder) TidyMVS() error { // Load minimal root module - err := mdr.LoadMinimalFromFS(".") + err := mdr.LoadMetaFromFS(".") if err != nil { return err } diff --git a/lib/modder/modder_vendor.go b/lib/modder/modder_vendor.go index 442727c..7ead73d 100644 --- a/lib/modder/modder_vendor.go +++ b/lib/modder/modder_vendor.go @@ -66,12 +66,18 @@ func (mdr *Modder) VendorMVS() error { var err error // Load minimal root module - err = mdr.LoadMinimalFromFS(".") + err = mdr.LoadMetaFromFS(".") if err != nil { fmt.Println(err) // return err } + + + + + return nil + err = mdr.CheckAndFetchRootDeps() if err != nil { fmt.Println(err) diff --git a/lib/modder/modder_verify.go b/lib/modder/modder_verify.go index ec210fe..c5514e5 100644 --- a/lib/modder/modder_verify.go +++ b/lib/modder/modder_verify.go @@ -32,17 +32,43 @@ func (mdr *Modder) Verify() error { // The entrypoint to the MVS internal verify process func (mdr *Modder) VerifyMVS() error { + valid := true + // Load minimal root module - err := mdr.LoadMinimalFromFS(".") + err := mdr.LoadMetaFromFS(".") if err != nil { return err } // Load the root module's deps - err = mdr.CompareModToSum() + present, missing, err := mdr.FindPresentMissingInSum() if err != nil { return err } + // Invalid if there are missing deps + if len(missing) > 0 { + valid = false + } + + for _, m := range missing { + R := mdr.module.SelfDeps[m] + fmt.Printf("Sumfile missing: %s@%s\n", R.NewPath, R.NewVersion) + err := fmt.Errorf("Sumfile missing: %s@%s", R.NewPath, R.NewVersion) + mdr.errors = append(mdr.errors, err) + } + + fmt.Println("Present\n-----------------") + for _, p := range present { + R := mdr.module.SelfDeps[p] + fmt.Println(R.NewPath, R.NewVersion) + } + fmt.Println("-----------------") + + + if !valid { + + return fmt.Errorf("Vendoring is in an inconsistent state, please run 'mvs vendor %s' ", mdr.Name) + } return nil } diff --git a/lib/modder/modder_write.go b/lib/modder/modder_write.go index 1b6a2d5..64a8347 100644 --- a/lib/modder/modder_write.go +++ b/lib/modder/modder_write.go @@ -53,7 +53,7 @@ func (mdr *Modder) WriteVendor() error { // Found one! if strings.HasPrefix(strings.ToUpper(file.Name()), fn) { // TODO, these functions should just take 2 billy FS - err = util.BillyCopyFile(baseDir, "/"+file.Name(), m.FS) + err = util.BillyWriteFileToOS(baseDir, "/"+file.Name(), m.FS) if err != nil { return err } @@ -65,7 +65,7 @@ func (mdr *Modder) WriteVendor() error { if len(mdr.VendorIncludeGlobs) > 0 || len(mdr.VendorExcludeGlobs) > 0 { // Just copy everything // TODO, these functions should just take 2 billy FS - err = util.BillyGlobCopy(baseDir, "/", m.FS, mdr.VendorIncludeGlobs, mdr.VendorExcludeGlobs) + err = util.BillyGlobWriteDirToOS(baseDir, "/", m.FS, mdr.VendorIncludeGlobs, mdr.VendorExcludeGlobs) if err != nil { return err } @@ -73,7 +73,7 @@ func (mdr *Modder) WriteVendor() error { } else { // Just copy everything // TODO, these functions should just take 2 billy FS - err = util.BillyCopyDir(baseDir, "/", m.FS) + err = util.BillyWriteDirToOS(baseDir, "/", m.FS) if err != nil { return err } diff --git a/lib/parse/sumfile/sumfile.go b/lib/parse/sumfile/sumfile.go index 840b6e3..d451ec4 100644 --- a/lib/parse/sumfile/sumfile.go +++ b/lib/parse/sumfile/sumfile.go @@ -58,7 +58,17 @@ func ParseSum(data []byte, file string) (Sum, error) { return sum, nil } -func (sum *Sum) Print() error { +func (sum *Sum) Add(ver Version, hash string) { + val, ok := sum.Mods[ver] + if !ok { + val = make([]string,0) + } + val = append(val, hash) + sum.Mods[ver] = val +} + +func (sum *Sum) Write() (string, error) { + var w strings.Builder // build up slice var sorted []Version for ver, _ := range sum.Mods { @@ -67,14 +77,17 @@ func (sum *Sum) Print() error { // sort slice by ver.Path sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Path == sorted[j].Path { + return sorted[i].Version < sorted[j].Version + } return sorted[i].Path < sorted[j].Path }) // print for _, ver := range sorted { list := sum.Mods[ver] - fmt.Println(ver.Path, ver.Version, list) + fmt.Fprintln(&w, ver.Path, ver.Version, strings.Join(list, " ")) } - return nil + return w.String(), nil } diff --git a/lib/repos/github/client.go b/lib/repos/github/client.go new file mode 100644 index 0000000..e0340cf --- /dev/null +++ b/lib/repos/github/client.go @@ -0,0 +1,27 @@ +package github + +import ( + "context" + "os" + + "golang.org/x/oauth2" + + "github.com/google/go-github/v30/github" +) + +func NewClient() (client *github.Client, err error) { + ctx := context.Background() + + if token := os.Getenv("GITHUB_TOKEN"); token != "" { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + client = github.NewClient(tc) + + } else { + client = github.NewClient(nil) + } + + return client, err +} diff --git a/lib/repos/github/tags.go b/lib/repos/github/tags.go new file mode 100644 index 0000000..0260742 --- /dev/null +++ b/lib/repos/github/tags.go @@ -0,0 +1,66 @@ +package github + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "strings" + + "github.com/google/go-github/v30/github" + "github.com/parnurzeal/gorequest" +) + +func GetTagsSplit(client *github.Client, module string) ([]*github.RepositoryTag, error) { + flds := strings.SplitN(module, "/", 1) + domain, rest := flds[0], flds[1] + + if domain != "github.com" { + return nil, fmt.Errorf("Github Tags Fetch called with non 'github.com' domain %q", module) + } + + flds = strings.Split(rest, "/") + owner, repo := flds[0], flds[1] + tags, _, err := client.Repositories.ListTags(context.Background(), owner, repo, nil) + if err != nil { + return nil, err + } + return tags, nil +} + +func GetTags(client *github.Client, owner, repo string) ([]*github.RepositoryTag, error) { + tags, _, err := client.Repositories.ListTags(context.Background(), owner, repo, nil) + if err != nil { + return nil, err + } + return tags, nil +} + +func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader, error) { + + url := *tag.ZipballURL + + req := gorequest.New().Get(url) + resp, data, errs := req.EndBytes() + + check := "http2: server sent GOAWAY and closed the connection" + if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { + fmt.Println("errs:", errs) + fmt.Println("resp:", resp) + fmt.Println("body:", len(data)) + return nil, errs[0] + } + + if len(errs) != 0 || resp.StatusCode >= 500 { + return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("Bad Request: " + string(resp.StatusCode)) + } + + r := bytes.NewReader(data) + + zfile, err := zip.NewReader(r, int64(len(data))) + + return zfile, err +} diff --git a/lib/repos/tags.go b/lib/repos/tags.go new file mode 100644 index 0000000..2da368c --- /dev/null +++ b/lib/repos/tags.go @@ -0,0 +1,19 @@ +package repos + +func IndexModule(module, version string) error { + domain, rest := strings.SplitN(module, "/", 1) + flds := string.Split(rest, "/") + owner, repo = flds[0], flds[1] + + fmt.Println("IndexModule:", module, version) + fmt.Println(" ", domain, owner, repo, version) + + switch domain { + case "github.com": + + default: + // Assume git + } + + return nil +} diff --git a/lib/util/billy.go b/lib/util/billy.go index 46c4c34..dbce050 100644 --- a/lib/util/billy.go +++ b/lib/util/billy.go @@ -1,12 +1,15 @@ package util import ( + "archive/zip" + "io" "io/ioutil" "os" "path" + "strings" - "github.com/bmatcuk/doublestar" "github.com/go-git/go-billy/v5" + "golang.org/x/mod/sumdb/dirhash" ) func BillyReadAllString(filename string, FS billy.Filesystem) (string, error) { @@ -27,8 +30,8 @@ func BillyReadAll(filename string, FS billy.Filesystem) ([]byte, error) { return ioutil.ReadAll(f) } -// Copies dir in FS onto the os filesystem at baseDir -func BillyCopyDir(baseDir string, dir string, FS billy.Filesystem) error { +// Writes dir in FS onto the os filesystem at baseDir +func BillyWriteDirToOS(baseDir string, dir string, FS billy.Filesystem) error { // fmt.Println("DIR: ", baseDir, dir) files, err := FS.ReadDir(dir) if err != nil { @@ -37,16 +40,16 @@ func BillyCopyDir(baseDir string, dir string, FS billy.Filesystem) error { for _, file := range files { longname := path.Join(dir, file.Name()) - // fmt.Println("DIR: ", baseDir, dir, file.Name(), longname, outname) + // fmt.Println("DIR: ", baseDir, dir, file.Name(), longname) if file.IsDir() { - err = BillyCopyDir(baseDir, longname, FS) + err = BillyWriteDirToOS(baseDir, longname, FS) if err != nil { return err } } else { - err = BillyCopyFile(baseDir, longname, FS) + err = BillyWriteFileToOS(baseDir, longname, FS) if err != nil { return err } @@ -57,36 +60,36 @@ func BillyCopyDir(baseDir string, dir string, FS billy.Filesystem) error { return nil } -// Copies file in FS onto the os filesystem at baseDir -func BillyCopyFile(baseDir string, file string, FS billy.Filesystem) error { +// Writes file in FS onto the os filesystem at baseDir +func BillyWriteFileToOS(baseDir string, file string, FS billy.Filesystem) error { outName := path.Join(baseDir, file) + // fmt.Println("FILE: ", outName) err := os.MkdirAll(path.Dir(outName), 0755) if err != nil { return err } - bf, err := FS.Open(file) + src, err := FS.Open(file) if err != nil { return err } + defer src.Close() - content, err := ioutil.ReadAll(bf) + dst, err := os.Create(outName) if err != nil { return err } + defer dst.Close() - err = ioutil.WriteFile(outName, content, 0644) - if err != nil { - return err - } + io.Copy(dst, src) return nil } -// Copies dir in FS onto the os filesystem at baseDir +// Write dir in FS onto the os filesystem at baseDir // -func BillyGlobCopy(baseDir string, dir string, FS billy.Filesystem, includes, excludes []string) error { +func BillyGlobWriteDirToOS(baseDir string, dir string, FS billy.Filesystem, includes, excludes []string) error { // fmt.Println("DIR: ", baseDir, dir) files, err := FS.ReadDir(dir) if err != nil { @@ -99,54 +102,158 @@ func BillyGlobCopy(baseDir string, dir string, FS billy.Filesystem, includes, ex // fmt.Println("GLOB? ", longname) if file.IsDir() { - err = BillyGlobCopy(baseDir, longname, FS, includes, excludes) + err = BillyGlobWriteDirToOS(baseDir, longname, FS, includes, excludes) if err != nil { return err } } else { - include := false - if len(includes) > 0 { - for _, pattern := range includes { - include, err = doublestar.PathMatch(pattern, longname) - // fmt.Println("GLOB++ ", longname, pattern, include) - if err != nil { - return err - } - if include { - break - } + include, _ := CheckShouldInclude(longname, includes, excludes) + // fmt.Println("COPY ==>", longname, include, exclude, include && !exclude) + + if include { + err = BillyWriteFileToOS(baseDir, longname, FS) + if err != nil { + return err } - } else { - include = true } - exclude := false - if len(excludes) > 0 { - for _, pattern := range excludes { - exclude, err = doublestar.PathMatch(pattern, longname) - // fmt.Println("GLOB-- ", longname, pattern, exclude) - if err != nil { - return err - } - if exclude { - break - } - } + } + } + + return nil +} + +func BillyLoadFromZip(zReader *zip.Reader, FS billy.Filesystem, trimFirstDir bool) error { + for _, f := range zReader.File { + + // Is this a directory? + if strings.HasSuffix(f.Name, "/") { + err := FS.MkdirAll(f.Name, 0755) + if err != nil { + return err } + continue + } - // fmt.Println("COPY ==>", longname, include, exclude, include && !exclude) + fn := f.Name[strings.Index(f.Name, "/")+1:] - if include && !exclude { - err = BillyCopyFile(baseDir, longname, FS) - if err != nil { - return err - } + src, err := f.Open() + if err != nil { + return err + } + defer src.Close() + + dst, err := FS.Create(fn) + if err != nil { + return err + } + defer dst.Close() + + _, err = io.Copy(dst, src) + if err != nil { + return err + } + + // fmt.Println("FromZip:", fn, cnt) + } + + return nil +} + +// Loads an initialized zip.Reader into an initialize billy.Filesystem +func BillyGlobLoadFromZip(zReader *zip.Reader, FS billy.Filesystem, includes, excludes []string) error { + + for _, f := range zReader.File { + include, _ := CheckShouldInclude(f.Name, includes, excludes) + if !include { + continue + } + + // Is this a directory? + if strings.HasSuffix(f.Name, "/") { + err := FS.MkdirAll(f.Name, 0755) + if err != nil { + return err } + continue + } + + fn := f.Name[strings.Index(f.Name, "/")+1:] + + src, err := f.Open() + if err != nil { + return err + } + defer src.Close() + + dst, err := FS.Create(fn) + if err != nil { + return err + } + defer dst.Close() + _, err = io.Copy(dst, src) + if err != nil { + return err } + } return nil } + +func BillyFilenames(dir string, FS billy.Filesystem) ([]string, error) { + out := []string{} + + files, err := FS.ReadDir(dir) + if err != nil { + return out, err + } + + for _, file := range files { + fullname := path.Join(dir, file.Name()) + + if file.IsDir() { + tmp, err := BillyFilenames(fullname, FS) + if err != nil { + return out, err + } + out = append(out, tmp...) + } else { + out = append(out, fullname) + } + } + + return out, nil +} + +func BillyCalcHash(FS billy.Filesystem) (string, error) { + return BillyCalcDirHash("/", FS) +} + +func BillyCalcDirHash(dir string, FS billy.Filesystem) (string, error) { + files, err := BillyFilenames(dir, FS) + if err != nil { + return "", err + } + + open := func (fn string) (io.ReadCloser, error) { + return FS.Open(fn) + } + + return dirhash.Hash1(files, open) +} + +func BillyCalcFileHash(filename string, FS billy.Filesystem) (string, error) { + if !strings.HasPrefix(filename, "/") { + filename = "/" + filename + } + + open := func (fn string) (io.ReadCloser, error) { + return FS.Open(fn) + } + + return dirhash.Hash1([]string{filename}, open) +} diff --git a/lib/util/glob.go b/lib/util/glob.go new file mode 100644 index 0000000..c8a4910 --- /dev/null +++ b/lib/util/glob.go @@ -0,0 +1,39 @@ +package util + +import ( + "github.com/bmatcuk/doublestar" +) + +func CheckShouldInclude(filename string, includes, excludes []string) (bool, error) { + var err error + include := false + + if len(includes) > 0 { + for _, pattern := range includes { + include, err = doublestar.PathMatch(pattern, filename) + if err != nil { + return false, err + } + if include { + break + } + } + } else { + include = true + } + + exclude := false + if len(excludes) > 0 { + for _, pattern := range excludes { + exclude, err = doublestar.PathMatch(pattern, filename) + if err != nil { + return false, err + } + if exclude { + break + } + } + } + + return include && !exclude, nil +}