Skip to content
This repository has been archived by the owner on Jun 5, 2020. It is now read-only.

Commit

Permalink
building blocks and parts of the whole
Browse files Browse the repository at this point in the history
  • Loading branch information
verdverm committed Apr 11, 2020
1 parent bb5ccc6 commit e3a82fe
Show file tree
Hide file tree
Showing 25 changed files with 734 additions and 75 deletions.
2 changes: 2 additions & 0 deletions cue.sums
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
github.com/hofstadter-io/hofmod-cli v0.3.0 h1:+wCqHV08voZyt87uKKFpNOst6wON+OFOEqTViIF4Enk=
github.com/hofstadter-io/hofmod-cli v0.3.0/cue.mods h1:hvuTtdDNMFf5D9Jv0SfQfg2qRBOkFGb+EVOwkVtYKQI=
35 changes: 35 additions & 0 deletions docs/process.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# MVS Processing

This is pseudo code for how MVS handles dependencies

From: `mvs vendor <lang>` (when no lang, loops over discovered)

### 1. Read Root Module

1. Read MVS / mod files
1. `.mvsconfig` if exists (this will configure paths and behaviors for this module)
1. `<lang>.mod` file, EXIT if does not exist, warn to init
1. `<lang>.sum` file, if exists
1. If compare(sum, mod) == Same / Valid
1. Check content is ok
1. If not, EXIT and WARN
1. For each entry in {mod - sum}
1. Fetch: cache -> remote
1. CALL: ReadDepsModule





### FN: ReadDepsModule


1. Read MVS / mod files
1. `.mvsconfig` if exists (this will configure paths and behaviors for this module)
1. `<lang>.mod` file, EXIT if does not exist, warn to init
1. `<lang>.sum` file, if exists
1.




7 changes: 7 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,15 @@ require (
github.com/bmatcuk/doublestar v1.2.2
github.com/go-git/go-billy/v5 v5.0.0
github.com/go-git/go-git/v5 v5.0.0
github.com/google/go-github/v30 v30.1.0
github.com/parnurzeal/gorequest v0.2.16
github.com/spf13/cobra v0.0.6
github.com/spf13/viper v1.4.0 // indirect
golang.org/x/mod v0.2.0
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
moul.io/http2curl v1.0.0 // indirect
)

// replace github.com/hofstadter-io/go-utils => ../go-utils
14 changes: 14 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,18 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo=
github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
Expand Down Expand Up @@ -101,6 +109,8 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuH
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -172,6 +182,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -199,6 +210,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
Expand All @@ -220,3 +232,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
33 changes: 33 additions & 0 deletions lib/cache/checksum.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package cache

import (
"fmt"
"path/filepath"
"os"

"golang.org/x/mod/sumdb/dirhash"
)

func Checksum(lang, mod, ver string) (string, error) {

dir := filepath.Join(
LocalCacheBaseDir,
"mod",
lang,
mod,
"@",
ver,
)

fmt.Println("Cache Load:", dir)

_, err := os.Lstat(dir)
if err != nil {
return "", err
}

h, err := dirhash.HashDir(dir, mod, dirhash.Hash1)

return h, err
}

43 changes: 43 additions & 0 deletions lib/cache/local.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package cache

import (
"path/filepath"

"github.com/hofstadter-io/mvs/lib/util"
)

var (
LocalCacheBaseDir = filepath.Join(util.UserHomeDir(), ".mvs")
)

func SetBaseDir(basedir string) {
LocalCacheBaseDir = basedir
}

/*
func Lookup(modFile, mdr, mod, ver string) (zdata []byte, err error) {
dir := filepath.Join(
LocalCacheBaseDir,
"mod",
mdr,
mod,
"@",
ver,
)
fmt.Println("Cache Lookup:", dir)
// TODO, lookup
var buf bytes.Buffer
m := module.Version{ Path: mod, Version: ver }
err = zip.CreateFromDir(&buf, m, dir, modFile, []string{"**.*"}, nil)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
*/
33 changes: 33 additions & 0 deletions lib/cache/lookup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package cache

import (
"fmt"
"path/filepath"
"os"

"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
)

func Load(lang, mod, ver string) (FS billy.Filesystem, err error) {

dir := filepath.Join(
LocalCacheBaseDir,
"mod",
lang,
mod,
"@",
ver,
)

fmt.Println("Cache Load:", dir)

_, err = os.Lstat(dir)
if err != nil {
return nil, err
}

FS = osfs.New(dir)

return FS, nil
}
22 changes: 22 additions & 0 deletions lib/cache/write.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package cache

import (
"path/filepath"

"github.com/go-git/go-billy/v5"

"github.com/hofstadter-io/mvs/lib/util"
)

func Write(lang, remote, owner, repo, tag string, FS billy.Filesystem) error {
outdir := filepath.Join(
LocalCacheBaseDir,
"mod",
lang,
remote,
owner,
repo + "@" + tag,
)

return util.BillyWriteDirToOS(outdir, "/", FS)
}
94 changes: 94 additions & 0 deletions lib/hack.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,104 @@ package lib

import (
"fmt"
"io/ioutil"
"strings"

googithub "github.com/google/go-github/v30/github"
"github.com/go-git/go-billy/v5/memfs"

"github.com/hofstadter-io/mvs/lib/cache"
"github.com/hofstadter-io/mvs/lib/parse/sumfile"
"github.com/hofstadter-io/mvs/lib/repos/github"
"github.com/hofstadter-io/mvs/lib/util"
)

func Hack(lang string, args []string) error {
fmt.Println("Hack", args)

client, err := github.NewClient()
if err != nil {
return err
}

owner := args[0]
repo := args[1]
tag := args[2]

tags, err := github.GetTags(client, owner, repo)
if err != nil {
return err
}

// The tag we are looking for
var T *googithub.RepositoryTag
for _, t := range tags {
if tag != "" && tag == *t.Name {
T = t
fmt.Printf("FOUND ")
}
fmt.Println(*t.Name, *t.Commit.SHA)
}

// Fetch and write to cache if tag found
if T != nil {
zReader, err := github.FetchTagZip(client, T)
if err != nil {
return fmt.Errorf("While fetching zipfile\n%w\n", err)
}
FS := memfs.New()

err = util.BillyLoadFromZip(zReader, FS, true)
if err != nil {
return fmt.Errorf("While reading zipfile\n%w\n", err)
}

// fmt.Println("GOT HERE 1")

err = cache.Write("hof", "github.com", owner, repo, tag, FS)
if err != nil {
return fmt.Errorf("While writing to cache\n%w\n", err)
}

// fmt.Println("GOT HERE 2")

dirhash, err := util.BillyCalcHash(FS)
if err != nil {
return fmt.Errorf("While calculating dir hash\n%w\n", err)
}

modhash, err := util.BillyCalcFileHash("cue.mods", FS)
if err != nil {
return fmt.Errorf("While calculating mod hash\n%w\n", err)
}

S := sumfile.Sum{
Mods: make(map[sumfile.Version][]string),
}

dver := sumfile.Version{
Path: strings.Join([]string{"github.com", owner, repo}, "/"),
Version: tag,
}
S.Add(dver, dirhash)

mver := sumfile.Version{
Path: strings.Join([]string{"github.com", owner, repo}, "/"),
Version: strings.Join([]string{tag, "cue.mods"}, "/"),
}
S.Add(mver, modhash)

fmt.Println("=====")
out, err := S.Write()
if err != nil {
return err
}
fmt.Println(out)
fmt.Println("=====")

ioutil.WriteFile("cue.sums", []byte(out), 0644)

}

return nil
}
29 changes: 29 additions & 0 deletions lib/modder/modder_check.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ func (mdr *Modder) CheckAndFetchRootDeps() error {
continue
}

// TODO Lookup in cache

// HANDLE remote and non-local replace the same way
ref, refs, err := git.IndexGitRemote(R.NewPath, R.NewVersion)
if err != nil {
Expand Down Expand Up @@ -251,6 +253,33 @@ func (mdr *Modder) CompareModToSum() error {
return nil
}

func (mdr *Modder) FindPresentMissingInSum() ([]string, []string, error) {
present := []string{}
missing := []string{}

mod := mdr.module
sf := mod.SumFile
if sf == nil {
return nil, nil, fmt.Errorf("No sum file %q for %s, run 'mvs vendor [%s]' to fix.", mdr.SumFile, mdr.Name, mdr.Name)
}

for path, R := range mod.SelfDeps {
ver := sumfile.Version{
Path: path,
Version: R.NewVersion,
}

_, ok := sf.Mods[ver]
if ok {
present = append(present, path)
} else {
missing = append(missing, path)
}
}

return present, missing, nil
}

func (mdr *Modder) CompareSumToVendor() error {

return nil
Expand Down
Loading

0 comments on commit e3a82fe

Please sign in to comment.