From 2197f208f607f0c3dc425ce88b9533a07276786e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Wed, 17 Oct 2018 12:57:43 +0200 Subject: [PATCH 1/4] Upgrade go-git dependency to v4.7.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Martín --- Gopkg.lock | 7 +- vendor/gopkg.in/src-d/go-git.v4/blame.go | 21 +- .../gopkg.in/src-d/go-git.v4/config/config.go | 5 + .../src-d/go-git.v4/config/refspec.go | 2 +- .../gopkg.in/src-d/go-git.v4/object_walker.go | 2 + vendor/gopkg.in/src-d/go-git.v4/options.go | 50 +- .../go-git.v4/plumbing/cache/buffer_lru.go | 98 ++++ .../src-d/go-git.v4/plumbing/cache/common.go | 13 + .../go-git.v4/plumbing/cache/object_lru.go | 24 +- .../plumbing/format/diff/unified_encoder.go | 8 +- .../plumbing/format/gitignore/pattern.go | 3 + .../plumbing/format/idxfile/decoder.go | 109 ++-- .../plumbing/format/idxfile/encoder.go | 101 ++-- .../plumbing/format/idxfile/idxfile.go | 345 +++++++++-- .../plumbing/format/idxfile/writer.go | 186 ++++++ .../plumbing/format/index/decoder.go | 2 +- .../plumbing/format/packfile/common.go | 20 +- .../plumbing/format/packfile/decoder.go | 495 ---------------- .../plumbing/format/packfile/fsobject.go | 116 ++++ .../plumbing/format/packfile/index.go | 115 ---- .../plumbing/format/packfile/packfile.go | 538 ++++++++++++++++++ .../plumbing/format/packfile/parser.go | 489 ++++++++++++++++ .../plumbing/format/packfile/patch_delta.go | 2 +- .../src-d/go-git.v4/plumbing/memory.go | 8 +- .../src-d/go-git.v4/plumbing/object/change.go | 21 +- .../src-d/go-git.v4/plumbing/object/commit.go | 68 ++- .../plumbing/object/commit_walker_file.go | 115 ++++ .../go-git.v4/plumbing/object/difftree.go | 13 +- .../src-d/go-git.v4/plumbing/object/object.go | 6 +- .../src-d/go-git.v4/plumbing/object/patch.go | 32 +- .../src-d/go-git.v4/plumbing/object/tag.go | 15 +- .../src-d/go-git.v4/plumbing/object/tree.go | 36 +- .../src-d/go-git.v4/plumbing/storer/object.go | 2 + .../plumbing/transport/http/common.go | 25 + .../transport/internal/common/common.go | 5 + .../plumbing/transport/server/loader.go | 3 +- .../plumbing/transport/ssh/auth_method.go | 14 +- vendor/gopkg.in/src-d/go-git.v4/prune.go | 2 +- vendor/gopkg.in/src-d/go-git.v4/references.go | 4 +- vendor/gopkg.in/src-d/go-git.v4/remote.go | 44 +- vendor/gopkg.in/src-d/go-git.v4/repository.go | 283 +++++++-- vendor/gopkg.in/src-d/go-git.v4/status.go | 15 +- .../storage/filesystem/dotgit/dotgit.go | 306 +++++++++- .../dotgit/dotgit_rewrite_packed_refs.go | 81 +++ .../dotgit/dotgit_rewrite_packed_refs_nix.go | 17 - .../dotgit_rewrite_packed_refs_norwfs.go | 34 -- .../dotgit_rewrite_packed_refs_windows.go | 42 -- .../storage/filesystem/dotgit/writers.go | 30 +- .../go-git.v4/storage/filesystem/module.go | 3 +- .../go-git.v4/storage/filesystem/object.go | 298 +++++++--- .../go-git.v4/storage/filesystem/storage.go | 38 +- .../src-d/go-git.v4/storage/memory/storage.go | 10 + .../src-d/go-git.v4/utils/diff/diff.go | 4 +- .../go-git.v4/utils/merkletrie/difftree.go | 21 + vendor/gopkg.in/src-d/go-git.v4/worktree.go | 56 +- .../src-d/go-git.v4/worktree_commit.go | 40 ++ 56 files changed, 3326 insertions(+), 1116 deletions(-) create mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/buffer_lru.go create mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/writer.go delete mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/decoder.go create mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/fsobject.go delete mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/index.go create mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/packfile.go create mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/parser.go create mode 100644 vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_walker_file.go create mode 100644 vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go delete mode 100644 vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go delete mode 100644 vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go delete mode 100644 vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go diff --git a/Gopkg.lock b/Gopkg.lock index 3aaee2b3..38fb32b9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -655,7 +655,7 @@ version = "v3.1.0" [[projects]] - digest = "1:f83a1e160ab0e7184716c36431aa12ffa7da8016cc9f2c00adc3a2fc77559583" + digest = "1:8225fd93ba0a18906b05a2ab52808349968f11a8c9c4be27001c2d10dbf781ef" name = "gopkg.in/src-d/go-git.v4" packages = [ ".", @@ -700,8 +700,8 @@ "utils/merkletrie/noder", ] pruneopts = "NUT" - revision = "b23570073eaee3489e5e3d666f22ba5cbeb53243" - version = "v4.4.1" + revision = "cd64b4d630b6c2d2b3d72e9615e14f9d58bb5787" + version = "v4.7.1" [[projects]] digest = "1:c1d9b0ec7eca0c5f1b82abe29ec426a94f43b0f1a4641f99ac80af921fe123fe" @@ -820,6 +820,7 @@ "gopkg.in/src-d/go-git.v4/plumbing", "gopkg.in/src-d/go-git.v4/plumbing/object", "gopkg.in/src-d/go-git.v4/plumbing/storer", + "gopkg.in/src-d/go-git.v4/plumbing/transport/http", "gopkg.in/src-d/go-git.v4/storage", "gopkg.in/src-d/go-git.v4/storage/filesystem", "gopkg.in/src-d/go-git.v4/utils/binary", diff --git a/vendor/gopkg.in/src-d/go-git.v4/blame.go b/vendor/gopkg.in/src-d/go-git.v4/blame.go index 349cdd9b..adb72d57 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/blame.go +++ b/vendor/gopkg.in/src-d/go-git.v4/blame.go @@ -123,14 +123,25 @@ func newLine(author, text string, date time.Time, hash plumbing.Hash) *Line { } func newLines(contents []string, commits []*object.Commit) ([]*Line, error) { - if len(contents) != len(commits) { - return nil, errors.New("contents and commits have different length") + lcontents := len(contents) + lcommits := len(commits) + + if lcontents != lcommits { + if lcontents == lcommits-1 && contents[lcontents-1] != "\n" { + contents = append(contents, "\n") + } else { + return nil, errors.New("contents and commits have different length") + } } - result := make([]*Line, 0, len(contents)) + + result := make([]*Line, 0, lcontents) for i := range contents { - l := newLine(commits[i].Author.Email, contents[i], commits[i].Author.When, commits[i].Hash) - result = append(result, l) + result = append(result, newLine( + commits[i].Author.Email, contents[i], + commits[i].Author.When, commits[i].Hash, + )) } + return result, nil } diff --git a/vendor/gopkg.in/src-d/go-git.v4/config/config.go b/vendor/gopkg.in/src-d/go-git.v4/config/config.go index ce6506da..a637f6d7 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/config/config.go +++ b/vendor/gopkg.in/src-d/go-git.v4/config/config.go @@ -40,6 +40,9 @@ type Config struct { IsBare bool // Worktree is the path to the root of the working tree. Worktree string + // CommentChar is the character indicating the start of a + // comment for commands like commit and tag + CommentChar string } Pack struct { @@ -113,6 +116,7 @@ const ( urlKey = "url" bareKey = "bare" worktreeKey = "worktree" + commentCharKey = "commentChar" windowKey = "window" mergeKey = "merge" @@ -151,6 +155,7 @@ func (c *Config) unmarshalCore() { } c.Core.Worktree = s.Options.Get(worktreeKey) + c.Core.CommentChar = s.Options.Get(commentCharKey) } func (c *Config) unmarshalPack() error { diff --git a/vendor/gopkg.in/src-d/go-git.v4/config/refspec.go b/vendor/gopkg.in/src-d/go-git.v4/config/refspec.go index c9b9d524..391705cc 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/config/refspec.go +++ b/vendor/gopkg.in/src-d/go-git.v4/config/refspec.go @@ -15,7 +15,7 @@ const ( var ( ErrRefSpecMalformedSeparator = errors.New("malformed refspec, separators are wrong") - ErrRefSpecMalformedWildcard = errors.New("malformed refspec, missmatched number of wildcards") + ErrRefSpecMalformedWildcard = errors.New("malformed refspec, mismatched number of wildcards") ) // RefSpec is a mapping from local branches to remote references diff --git a/vendor/gopkg.in/src-d/go-git.v4/object_walker.go b/vendor/gopkg.in/src-d/go-git.v4/object_walker.go index 4cbbcca6..f8b19cdb 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/object_walker.go +++ b/vendor/gopkg.in/src-d/go-git.v4/object_walker.go @@ -94,6 +94,8 @@ func (p *objectWalker) walkObjectTree(hash plumbing.Hash) error { return err } } + case *object.Tag: + return p.walkObjectTree(obj.Target) default: // Error out on unhandled object types. return fmt.Errorf("Unknown object %X %s %T\n", obj.ID(), obj.Type(), obj) diff --git a/vendor/gopkg.in/src-d/go-git.v4/options.go b/vendor/gopkg.in/src-d/go-git.v4/options.go index 885980ef..5d10a88c 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/options.go +++ b/vendor/gopkg.in/src-d/go-git.v4/options.go @@ -3,7 +3,9 @@ package git import ( "errors" "regexp" + "strings" + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -229,8 +231,9 @@ var ( // CheckoutOptions describes how a checkout 31operation should be performed. type CheckoutOptions struct { - // Hash to be checked out, if used HEAD will in detached mode. Branch and - // Hash are mutually exclusive, if Create is not used. + // Hash is the hash of the commit to be checked out. If used, HEAD will be + // in detached mode. If Create is not used, Branch and Hash are mutually + // exclusive. Hash plumbing.Hash // Branch to be checked out, if Branch and Hash are empty is set to `master`. Branch plumbing.ReferenceName @@ -328,6 +331,10 @@ type LogOptions struct { // set Order=LogOrderCommitterTime for ordering by committer time (more compatible with `git log`) // set Order=LogOrderBSF for Breadth-first search Order LogOrder + + // Show only those commits in which the specified file was inserted/updated. + // It is equivalent to running `git log -- `. + FileName *string } var ( @@ -347,6 +354,10 @@ type CommitOptions struct { // Parents are the parents commits for the new commit, by default when // len(Parents) is zero, the hash of HEAD reference is used. Parents []plumbing.Hash + // SignKey denotes a key to sign the commit with. A nil value here means the + // commit will not be signed. The private key must be present and already + // decrypted. + SignKey *openpgp.Entity } // Validate validates the fields and sets the default values. @@ -373,6 +384,41 @@ func (o *CommitOptions) Validate(r *Repository) error { return nil } +var ( + ErrMissingName = errors.New("name field is required") + ErrMissingTagger = errors.New("tagger field is required") + ErrMissingMessage = errors.New("message field is required") +) + +// CreateTagOptions describes how a tag object should be created. +type CreateTagOptions struct { + // Tagger defines the signature of the tag creator. + Tagger *object.Signature + // Message defines the annotation of the tag. It is canonicalized during + // validation into the format expected by git - no leading whitespace and + // ending in a newline. + Message string + // SignKey denotes a key to sign the tag with. A nil value here means the tag + // will not be signed. The private key must be present and already decrypted. + SignKey *openpgp.Entity +} + +// Validate validates the fields and sets the default values. +func (o *CreateTagOptions) Validate(r *Repository, hash plumbing.Hash) error { + if o.Tagger == nil { + return ErrMissingTagger + } + + if o.Message == "" { + return ErrMissingMessage + } + + // Canonicalize the message into the expected message format. + o.Message = strings.TrimSpace(o.Message) + "\n" + + return nil +} + // ListOptions describes how a remote list should be performed. type ListOptions struct { // Auth credentials, if required, to use with the remote repository. diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/buffer_lru.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/buffer_lru.go new file mode 100644 index 00000000..acaf1952 --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/buffer_lru.go @@ -0,0 +1,98 @@ +package cache + +import ( + "container/list" + "sync" +) + +// BufferLRU implements an object cache with an LRU eviction policy and a +// maximum size (measured in object size). +type BufferLRU struct { + MaxSize FileSize + + actualSize FileSize + ll *list.List + cache map[int64]*list.Element + mut sync.Mutex +} + +// NewBufferLRU creates a new BufferLRU with the given maximum size. The maximum +// size will never be exceeded. +func NewBufferLRU(maxSize FileSize) *BufferLRU { + return &BufferLRU{MaxSize: maxSize} +} + +// NewBufferLRUDefault creates a new BufferLRU with the default cache size. +func NewBufferLRUDefault() *BufferLRU { + return &BufferLRU{MaxSize: DefaultMaxSize} +} + +type buffer struct { + Key int64 + Slice []byte +} + +// Put puts a buffer into the cache. If the buffer is already in the cache, it +// will be marked as used. Otherwise, it will be inserted. A buffers might +// be evicted to make room for the new one. +func (c *BufferLRU) Put(key int64, slice []byte) { + c.mut.Lock() + defer c.mut.Unlock() + + if c.cache == nil { + c.actualSize = 0 + c.cache = make(map[int64]*list.Element, 1000) + c.ll = list.New() + } + + bufSize := FileSize(len(slice)) + if ee, ok := c.cache[key]; ok { + oldBuf := ee.Value.(buffer) + // in this case bufSize is a delta: new size - old size + bufSize -= FileSize(len(oldBuf.Slice)) + c.ll.MoveToFront(ee) + ee.Value = buffer{key, slice} + } else { + if bufSize > c.MaxSize { + return + } + ee := c.ll.PushFront(buffer{key, slice}) + c.cache[key] = ee + } + + c.actualSize += bufSize + for c.actualSize > c.MaxSize { + last := c.ll.Back() + lastObj := last.Value.(buffer) + lastSize := FileSize(len(lastObj.Slice)) + + c.ll.Remove(last) + delete(c.cache, lastObj.Key) + c.actualSize -= lastSize + } +} + +// Get returns a buffer by its key. It marks the buffer as used. If the buffer +// is not in the cache, (nil, false) will be returned. +func (c *BufferLRU) Get(key int64) ([]byte, bool) { + c.mut.Lock() + defer c.mut.Unlock() + + ee, ok := c.cache[key] + if !ok { + return nil, false + } + + c.ll.MoveToFront(ee) + return ee.Value.(buffer).Slice, true +} + +// Clear the content of this buffer cache. +func (c *BufferLRU) Clear() { + c.mut.Lock() + defer c.mut.Unlock() + + c.ll = nil + c.cache = nil + c.actualSize = 0 +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/common.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/common.go index e77baf01..2b7f36a5 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/common.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/common.go @@ -24,3 +24,16 @@ type Object interface { // Clear clears every object from the cache. Clear() } + +// Buffer is an interface to a buffer cache. +type Buffer interface { + // Put puts a buffer into the cache. If the buffer is already in the cache, + // it will be marked as used. Otherwise, it will be inserted. Buffer might + // be evicted to make room for the new one. + Put(key int64, slice []byte) + // Get returns a buffer by its key. It marks the buffer as used. If the + // buffer is not in the cache, (nil, false) will be returned. + Get(key int64) ([]byte, bool) + // Clear clears every object from the cache. + Clear() +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/object_lru.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/object_lru.go index 04945395..53d8b02d 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/object_lru.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/cache/object_lru.go @@ -42,20 +42,24 @@ func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { c.ll = list.New() } + objSize := FileSize(obj.Size()) key := obj.Hash() if ee, ok := c.cache[key]; ok { + oldObj := ee.Value.(plumbing.EncodedObject) + // in this case objSize is a delta: new size - old size + objSize -= FileSize(oldObj.Size()) c.ll.MoveToFront(ee) ee.Value = obj - return - } - - objSize := FileSize(obj.Size()) - - if objSize > c.MaxSize { - return + } else { + if objSize > c.MaxSize { + return + } + ee := c.ll.PushFront(obj) + c.cache[key] = ee } - for c.actualSize+objSize > c.MaxSize { + c.actualSize += objSize + for c.actualSize > c.MaxSize { last := c.ll.Back() lastObj := last.Value.(plumbing.EncodedObject) lastSize := FileSize(lastObj.Size()) @@ -64,10 +68,6 @@ func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { delete(c.cache, lastObj.Hash()) c.actualSize -= lastSize } - - ee := c.ll.PushFront(obj) - c.cache[key] = ee - c.actualSize += objSize } // Get returns an object by its hash. It marks the object as used. If the object diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/diff/unified_encoder.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/diff/unified_encoder.go index 58edd951..8bd6d8ab 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/diff/unified_encoder.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/diff/unified_encoder.go @@ -237,9 +237,13 @@ func (c *hunksGenerator) addLineNumbers(la, lb int, linesBefore int, i int, op O // we need to search for a reference for the next diff switch { case linesBefore != 0 && c.ctxLines != 0: - clb = lb - c.ctxLines + 1 + if lb > c.ctxLines { + clb = lb - c.ctxLines + 1 + } else { + clb = 1 + } case c.ctxLines == 0: - clb = lb - c.ctxLines + clb = lb case i != len(c.chunks)-1: next := c.chunks[i+1] if next.Type() == op || next.Type() == Equal { diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/gitignore/pattern.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/gitignore/pattern.go index 26033522..098cb502 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/gitignore/pattern.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/gitignore/pattern.go @@ -133,6 +133,9 @@ func (p *pattern) globMatch(path []string, isDir bool) bool { } else if match { matched = true break + } else if len(path) == 0 { + // if nothing left then fail + matched = false } } } else { diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/decoder.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/decoder.go index 45afb1ec..5b927826 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/decoder.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/decoder.go @@ -17,6 +17,11 @@ var ( ErrMalformedIdxFile = errors.New("Malformed IDX file") ) +const ( + fanout = 256 + objectIDLength = 20 +) + // Decoder reads and decodes idx files from an input stream. type Decoder struct { *bufio.Reader @@ -27,13 +32,13 @@ func NewDecoder(r io.Reader) *Decoder { return &Decoder{bufio.NewReader(r)} } -// Decode reads from the stream and decode the content into the Idxfile struct. -func (d *Decoder) Decode(idx *Idxfile) error { +// Decode reads from the stream and decode the content into the MemoryIndex struct. +func (d *Decoder) Decode(idx *MemoryIndex) error { if err := validateHeader(d); err != nil { return err } - flow := []func(*Idxfile, io.Reader) error{ + flow := []func(*MemoryIndex, io.Reader) error{ readVersion, readFanout, readObjectNames, @@ -48,10 +53,6 @@ func (d *Decoder) Decode(idx *Idxfile) error { } } - if !idx.isValid() { - return ErrMalformedIdxFile - } - return nil } @@ -68,7 +69,7 @@ func validateHeader(r io.Reader) error { return nil } -func readVersion(idx *Idxfile, r io.Reader) error { +func readVersion(idx *MemoryIndex, r io.Reader) error { v, err := binary.ReadUint32(r) if err != nil { return err @@ -82,74 +83,92 @@ func readVersion(idx *Idxfile, r io.Reader) error { return nil } -func readFanout(idx *Idxfile, r io.Reader) error { - var err error - for i := 0; i < 255; i++ { - idx.Fanout[i], err = binary.ReadUint32(r) +func readFanout(idx *MemoryIndex, r io.Reader) error { + for k := 0; k < fanout; k++ { + n, err := binary.ReadUint32(r) if err != nil { return err } + + idx.Fanout[k] = n + idx.FanoutMapping[k] = noMapping } - idx.ObjectCount, err = binary.ReadUint32(r) - return err + return nil } -func readObjectNames(idx *Idxfile, r io.Reader) error { - c := int(idx.ObjectCount) - new := make([]Entry, c) - for i := 0; i < c; i++ { - e := &new[i] - if _, err := io.ReadFull(r, e.Hash[:]); err != nil { +func readObjectNames(idx *MemoryIndex, r io.Reader) error { + for k := 0; k < fanout; k++ { + var buckets uint32 + if k == 0 { + buckets = idx.Fanout[k] + } else { + buckets = idx.Fanout[k] - idx.Fanout[k-1] + } + + if buckets == 0 { + continue + } + + if buckets < 0 { + return ErrMalformedIdxFile + } + + idx.FanoutMapping[k] = len(idx.Names) + + nameLen := int(buckets * objectIDLength) + bin := make([]byte, nameLen) + if _, err := io.ReadFull(r, bin); err != nil { return err } - idx.Entries = append(idx.Entries, e) + idx.Names = append(idx.Names, bin) + idx.Offset32 = append(idx.Offset32, make([]byte, buckets*4)) + idx.CRC32 = append(idx.CRC32, make([]byte, buckets*4)) } return nil } -func readCRC32(idx *Idxfile, r io.Reader) error { - c := int(idx.ObjectCount) - for i := 0; i < c; i++ { - if err := binary.Read(r, &idx.Entries[i].CRC32); err != nil { - return err +func readCRC32(idx *MemoryIndex, r io.Reader) error { + for k := 0; k < fanout; k++ { + if pos := idx.FanoutMapping[k]; pos != noMapping { + if _, err := io.ReadFull(r, idx.CRC32[pos]); err != nil { + return err + } } } return nil } -func readOffsets(idx *Idxfile, r io.Reader) error { - c := int(idx.ObjectCount) - - for i := 0; i < c; i++ { - o, err := binary.ReadUint32(r) - if err != nil { - return err +func readOffsets(idx *MemoryIndex, r io.Reader) error { + var o64cnt int + for k := 0; k < fanout; k++ { + if pos := idx.FanoutMapping[k]; pos != noMapping { + if _, err := io.ReadFull(r, idx.Offset32[pos]); err != nil { + return err + } + + for p := 0; p < len(idx.Offset32[pos]); p += 4 { + if idx.Offset32[pos][p]&(byte(1)<<7) > 0 { + o64cnt++ + } + } } - - idx.Entries[i].Offset = uint64(o) } - for i := 0; i < c; i++ { - if idx.Entries[i].Offset <= offsetLimit { - continue - } - - o, err := binary.ReadUint64(r) - if err != nil { + if o64cnt > 0 { + idx.Offset64 = make([]byte, o64cnt*8) + if _, err := io.ReadFull(r, idx.Offset64); err != nil { return err } - - idx.Entries[i].Offset = o } return nil } -func readChecksums(idx *Idxfile, r io.Reader) error { +func readChecksums(idx *MemoryIndex, r io.Reader) error { if _, err := io.ReadFull(r, idx.PackfileChecksum[:]); err != nil { return err } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/encoder.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/encoder.go index 40abfb83..e4795110 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/encoder.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/encoder.go @@ -4,12 +4,11 @@ import ( "crypto/sha1" "hash" "io" - "sort" "gopkg.in/src-d/go-git.v4/utils/binary" ) -// Encoder writes Idxfile structs to an output stream. +// Encoder writes MemoryIndex structs to an output stream. type Encoder struct { io.Writer hash hash.Hash @@ -22,11 +21,9 @@ func NewEncoder(w io.Writer) *Encoder { return &Encoder{mw, h} } -// Encode encodes an Idxfile to the encoder writer. -func (e *Encoder) Encode(idx *Idxfile) (int, error) { - idx.Entries.Sort() - - flow := []func(*Idxfile) (int, error){ +// Encode encodes an MemoryIndex to the encoder writer. +func (e *Encoder) Encode(idx *MemoryIndex) (int, error) { + flow := []func(*MemoryIndex) (int, error){ e.encodeHeader, e.encodeFanout, e.encodeHashes, @@ -48,7 +45,7 @@ func (e *Encoder) Encode(idx *Idxfile) (int, error) { return sz, nil } -func (e *Encoder) encodeHeader(idx *Idxfile) (int, error) { +func (e *Encoder) encodeHeader(idx *MemoryIndex) (int, error) { c, err := e.Write(idxHeader) if err != nil { return c, err @@ -57,75 +54,81 @@ func (e *Encoder) encodeHeader(idx *Idxfile) (int, error) { return c + 4, binary.WriteUint32(e, idx.Version) } -func (e *Encoder) encodeFanout(idx *Idxfile) (int, error) { - fanout := idx.calculateFanout() - for _, c := range fanout { +func (e *Encoder) encodeFanout(idx *MemoryIndex) (int, error) { + for _, c := range idx.Fanout { if err := binary.WriteUint32(e, c); err != nil { return 0, err } } - return 1024, nil + return fanout * 4, nil } -func (e *Encoder) encodeHashes(idx *Idxfile) (int, error) { - sz := 0 - for _, ent := range idx.Entries { - i, err := e.Write(ent.Hash[:]) - sz += i +func (e *Encoder) encodeHashes(idx *MemoryIndex) (int, error) { + var size int + for k := 0; k < fanout; k++ { + pos := idx.FanoutMapping[k] + if pos == noMapping { + continue + } + n, err := e.Write(idx.Names[pos]) if err != nil { - return sz, err + return size, err } + size += n } - - return sz, nil + return size, nil } -func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) { - sz := 0 - for _, ent := range idx.Entries { - err := binary.Write(e, ent.CRC32) - sz += 4 +func (e *Encoder) encodeCRC32(idx *MemoryIndex) (int, error) { + var size int + for k := 0; k < fanout; k++ { + pos := idx.FanoutMapping[k] + if pos == noMapping { + continue + } + n, err := e.Write(idx.CRC32[pos]) if err != nil { - return sz, err + return size, err } + + size += n } - return sz, nil + return size, nil } -func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) { - sz := 0 - - var o64bits []uint64 - for _, ent := range idx.Entries { - o := ent.Offset - if o > offsetLimit { - o64bits = append(o64bits, o) - o = offsetLimit + uint64(len(o64bits)) +func (e *Encoder) encodeOffsets(idx *MemoryIndex) (int, error) { + var size int + for k := 0; k < fanout; k++ { + pos := idx.FanoutMapping[k] + if pos == noMapping { + continue } - if err := binary.WriteUint32(e, uint32(o)); err != nil { - return sz, err + n, err := e.Write(idx.Offset32[pos]) + if err != nil { + return size, err } - sz += 4 + size += n } - for _, o := range o64bits { - if err := binary.WriteUint64(e, o); err != nil { - return sz, err + if len(idx.Offset64) > 0 { + n, err := e.Write(idx.Offset64) + if err != nil { + return size, err } - sz += 8 + size += n } - return sz, nil + return size, nil } -func (e *Encoder) encodeChecksums(idx *Idxfile) (int, error) { +func (e *Encoder) encodeChecksums(idx *MemoryIndex) (int, error) { if _, err := e.Write(idx.PackfileChecksum[:]); err != nil { return 0, err } @@ -137,11 +140,3 @@ func (e *Encoder) encodeChecksums(idx *Idxfile) (int, error) { return 40, nil } - -// EntryList implements sort.Interface allowing sorting in increasing order. -type EntryList []*Entry - -func (p EntryList) Len() int { return len(p) } -func (p EntryList) Less(i, j int) bool { return p[i].Hash.String() < p[j].Hash.String() } -func (p EntryList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p EntryList) Sort() { sort.Sort(p) } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/idxfile.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/idxfile.go index 6b05eaac..5fed278b 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/idxfile.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/idxfile.go @@ -1,30 +1,307 @@ package idxfile -import "gopkg.in/src-d/go-git.v4/plumbing" +import ( + "bytes" + "io" + "sort" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/utils/binary" +) const ( // VersionSupported is the only idx version supported. VersionSupported = 2 - offsetLimit = 0x7fffffff + noMapping = -1 ) var ( idxHeader = []byte{255, 't', 'O', 'c'} ) -// Idxfile is the in memory representation of an idx file. -type Idxfile struct { - Version uint32 - Fanout [255]uint32 - ObjectCount uint32 - Entries EntryList +// Index represents an index of a packfile. +type Index interface { + // Contains checks whether the given hash is in the index. + Contains(h plumbing.Hash) (bool, error) + // FindOffset finds the offset in the packfile for the object with + // the given hash. + FindOffset(h plumbing.Hash) (int64, error) + // FindCRC32 finds the CRC32 of the object with the given hash. + FindCRC32(h plumbing.Hash) (uint32, error) + // FindHash finds the hash for the object with the given offset. + FindHash(o int64) (plumbing.Hash, error) + // Count returns the number of entries in the index. + Count() (int64, error) + // Entries returns an iterator to retrieve all index entries. + Entries() (EntryIter, error) + // EntriesByOffset returns an iterator to retrieve all index entries ordered + // by offset. + EntriesByOffset() (EntryIter, error) +} + +// MemoryIndex is the in memory representation of an idx file. +type MemoryIndex struct { + Version uint32 + Fanout [256]uint32 + // FanoutMapping maps the position in the fanout table to the position + // in the Names, Offset32 and CRC32 slices. This improves the memory + // usage by not needing an array with unnecessary empty slots. + FanoutMapping [256]int + Names [][]byte + Offset32 [][]byte + CRC32 [][]byte + Offset64 []byte PackfileChecksum [20]byte IdxChecksum [20]byte + + offsetHash map[int64]plumbing.Hash +} + +var _ Index = (*MemoryIndex)(nil) + +// NewMemoryIndex returns an instance of a new MemoryIndex. +func NewMemoryIndex() *MemoryIndex { + return &MemoryIndex{} +} + +func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) (int, bool) { + k := idx.FanoutMapping[h[0]] + if k == noMapping { + return 0, false + } + + if len(idx.Names) <= k { + return 0, false + } + + data := idx.Names[k] + high := uint64(len(idx.Offset32[k])) >> 2 + if high == 0 { + return 0, false + } + + low := uint64(0) + for { + mid := (low + high) >> 1 + offset := mid * objectIDLength + + cmp := bytes.Compare(h[:], data[offset:offset+objectIDLength]) + if cmp < 0 { + high = mid + } else if cmp == 0 { + return int(mid), true + } else { + low = mid + 1 + } + + if low >= high { + break + } + } + + return 0, false +} + +// Contains implements the Index interface. +func (idx *MemoryIndex) Contains(h plumbing.Hash) (bool, error) { + _, ok := idx.findHashIndex(h) + return ok, nil +} + +// FindOffset implements the Index interface. +func (idx *MemoryIndex) FindOffset(h plumbing.Hash) (int64, error) { + if len(idx.FanoutMapping) <= int(h[0]) { + return 0, plumbing.ErrObjectNotFound + } + + k := idx.FanoutMapping[h[0]] + i, ok := idx.findHashIndex(h) + if !ok { + return 0, plumbing.ErrObjectNotFound + } + + return idx.getOffset(k, i) +} + +const isO64Mask = uint64(1) << 31 + +func (idx *MemoryIndex) getOffset(firstLevel, secondLevel int) (int64, error) { + offset := secondLevel << 2 + buf := bytes.NewBuffer(idx.Offset32[firstLevel][offset : offset+4]) + ofs, err := binary.ReadUint32(buf) + if err != nil { + return -1, err + } + + if (uint64(ofs) & isO64Mask) != 0 { + offset := 8 * (uint64(ofs) & ^isO64Mask) + buf := bytes.NewBuffer(idx.Offset64[offset : offset+8]) + n, err := binary.ReadUint64(buf) + if err != nil { + return -1, err + } + + return int64(n), nil + } + + return int64(ofs), nil +} + +// FindCRC32 implements the Index interface. +func (idx *MemoryIndex) FindCRC32(h plumbing.Hash) (uint32, error) { + k := idx.FanoutMapping[h[0]] + i, ok := idx.findHashIndex(h) + if !ok { + return 0, plumbing.ErrObjectNotFound + } + + return idx.getCRC32(k, i) +} + +func (idx *MemoryIndex) getCRC32(firstLevel, secondLevel int) (uint32, error) { + offset := secondLevel << 2 + buf := bytes.NewBuffer(idx.CRC32[firstLevel][offset : offset+4]) + return binary.ReadUint32(buf) +} + +// FindHash implements the Index interface. +func (idx *MemoryIndex) FindHash(o int64) (plumbing.Hash, error) { + // Lazily generate the reverse offset/hash map if required. + if idx.offsetHash == nil { + if err := idx.genOffsetHash(); err != nil { + return plumbing.ZeroHash, err + } + } + + hash, ok := idx.offsetHash[o] + if !ok { + return plumbing.ZeroHash, plumbing.ErrObjectNotFound + } + + return hash, nil +} + +// genOffsetHash generates the offset/hash mapping for reverse search. +func (idx *MemoryIndex) genOffsetHash() error { + count, err := idx.Count() + if err != nil { + return err + } + + idx.offsetHash = make(map[int64]plumbing.Hash, count) + + iter, err := idx.Entries() + if err != nil { + return err + } + + for { + entry, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + idx.offsetHash[int64(entry.Offset)] = entry.Hash + } +} + +// Count implements the Index interface. +func (idx *MemoryIndex) Count() (int64, error) { + return int64(idx.Fanout[fanout-1]), nil } -func NewIdxfile() *Idxfile { - return &Idxfile{} +// Entries implements the Index interface. +func (idx *MemoryIndex) Entries() (EntryIter, error) { + return &idxfileEntryIter{idx, 0, 0, 0}, nil +} + +// EntriesByOffset implements the Index interface. +func (idx *MemoryIndex) EntriesByOffset() (EntryIter, error) { + count, err := idx.Count() + if err != nil { + return nil, err + } + + iter := &idxfileEntryOffsetIter{ + entries: make(entriesByOffset, count), + } + + entries, err := idx.Entries() + if err != nil { + return nil, err + } + + for pos := 0; int64(pos) < count; pos++ { + entry, err := entries.Next() + if err != nil { + return nil, err + } + + iter.entries[pos] = entry + } + + sort.Sort(iter.entries) + + return iter, nil +} + +// EntryIter is an iterator that will return the entries in a packfile index. +type EntryIter interface { + // Next returns the next entry in the packfile index. + Next() (*Entry, error) + // Close closes the iterator. + Close() error +} + +type idxfileEntryIter struct { + idx *MemoryIndex + total int + firstLevel, secondLevel int +} + +func (i *idxfileEntryIter) Next() (*Entry, error) { + for { + if i.firstLevel >= fanout { + return nil, io.EOF + } + + if i.total >= int(i.idx.Fanout[i.firstLevel]) { + i.firstLevel++ + i.secondLevel = 0 + continue + } + + entry := new(Entry) + ofs := i.secondLevel * objectIDLength + copy(entry.Hash[:], i.idx.Names[i.idx.FanoutMapping[i.firstLevel]][ofs:]) + + pos := i.idx.FanoutMapping[entry.Hash[0]] + + offset, err := i.idx.getOffset(pos, i.secondLevel) + if err != nil { + return nil, err + } + entry.Offset = uint64(offset) + + entry.CRC32, err = i.idx.getCRC32(pos, i.secondLevel) + if err != nil { + return nil, err + } + + i.secondLevel++ + i.total++ + + return entry, nil + } +} + +func (i *idxfileEntryIter) Close() error { + i.firstLevel = fanout + return nil } // Entry is the in memory representation of an object entry in the idx file. @@ -34,35 +311,37 @@ type Entry struct { Offset uint64 } -// Add adds a new Entry with the given values to the Idxfile. -func (idx *Idxfile) Add(h plumbing.Hash, offset uint64, crc32 uint32) { - idx.Entries = append(idx.Entries, &Entry{ - Hash: h, - Offset: offset, - CRC32: crc32, - }) +type idxfileEntryOffsetIter struct { + entries entriesByOffset + pos int } -func (idx *Idxfile) isValid() bool { - fanout := idx.calculateFanout() - for k, c := range idx.Fanout { - if fanout[k] != c { - return false - } +func (i *idxfileEntryOffsetIter) Next() (*Entry, error) { + if i.pos >= len(i.entries) { + return nil, io.EOF } - return true + entry := i.entries[i.pos] + i.pos++ + + return entry, nil } -func (idx *Idxfile) calculateFanout() [256]uint32 { - fanout := [256]uint32{} - for _, e := range idx.Entries { - fanout[e.Hash[0]]++ - } +func (i *idxfileEntryOffsetIter) Close() error { + i.pos = len(i.entries) + 1 + return nil +} - for i := 1; i < 256; i++ { - fanout[i] += fanout[i-1] - } +type entriesByOffset []*Entry + +func (o entriesByOffset) Len() int { + return len(o) +} + +func (o entriesByOffset) Less(i int, j int) bool { + return o[i].Offset < o[j].Offset +} - return fanout +func (o entriesByOffset) Swap(i int, j int) { + o[i], o[j] = o[j], o[i] } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/writer.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/writer.go new file mode 100644 index 00000000..aa919e78 --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/idxfile/writer.go @@ -0,0 +1,186 @@ +package idxfile + +import ( + "bytes" + "fmt" + "math" + "sort" + "sync" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/utils/binary" +) + +// objects implements sort.Interface and uses hash as sorting key. +type objects []Entry + +// Writer implements a packfile Observer interface and is used to generate +// indexes. +type Writer struct { + m sync.Mutex + + count uint32 + checksum plumbing.Hash + objects objects + offset64 uint32 + finished bool + index *MemoryIndex + added map[plumbing.Hash]struct{} +} + +// Index returns a previously created MemoryIndex or creates a new one if +// needed. +func (w *Writer) Index() (*MemoryIndex, error) { + w.m.Lock() + defer w.m.Unlock() + + if w.index == nil { + return w.createIndex() + } + + return w.index, nil +} + +// Add appends new object data. +func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { + w.m.Lock() + defer w.m.Unlock() + + if w.added == nil { + w.added = make(map[plumbing.Hash]struct{}) + } + + if _, ok := w.added[h]; !ok { + w.added[h] = struct{}{} + w.objects = append(w.objects, Entry{h, crc, pos}) + } + +} + +func (w *Writer) Finished() bool { + return w.finished +} + +// OnHeader implements packfile.Observer interface. +func (w *Writer) OnHeader(count uint32) error { + w.count = count + w.objects = make(objects, 0, count) + return nil +} + +// OnInflatedObjectHeader implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { + return nil +} + +// OnInflatedObjectContent implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, _ []byte) error { + w.Add(h, uint64(pos), crc) + return nil +} + +// OnFooter implements packfile.Observer interface. +func (w *Writer) OnFooter(h plumbing.Hash) error { + w.checksum = h + w.finished = true + _, err := w.createIndex() + if err != nil { + return err + } + + return nil +} + +// creatIndex returns a filled MemoryIndex with the information filled by +// the observer callbacks. +func (w *Writer) createIndex() (*MemoryIndex, error) { + if !w.finished { + return nil, fmt.Errorf("the index still hasn't finished building") + } + + idx := new(MemoryIndex) + w.index = idx + + sort.Sort(w.objects) + + // unmap all fans by default + for i := range idx.FanoutMapping { + idx.FanoutMapping[i] = noMapping + } + + buf := new(bytes.Buffer) + + last := -1 + bucket := -1 + for i, o := range w.objects { + fan := o.Hash[0] + + // fill the gaps between fans + for j := last + 1; j < int(fan); j++ { + idx.Fanout[j] = uint32(i) + } + + // update the number of objects for this position + idx.Fanout[fan] = uint32(i + 1) + + // we move from one bucket to another, update counters and allocate + // memory + if last != int(fan) { + bucket++ + idx.FanoutMapping[fan] = bucket + last = int(fan) + + idx.Names = append(idx.Names, make([]byte, 0)) + idx.Offset32 = append(idx.Offset32, make([]byte, 0)) + idx.CRC32 = append(idx.CRC32, make([]byte, 0)) + } + + idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) + + offset := o.Offset + if offset > math.MaxInt32 { + offset = w.addOffset64(offset) + } + + buf.Truncate(0) + binary.WriteUint32(buf, uint32(offset)) + idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) + + buf.Truncate(0) + binary.WriteUint32(buf, uint32(o.CRC32)) + idx.CRC32[bucket] = append(idx.CRC32[bucket], buf.Bytes()...) + } + + for j := last + 1; j < 256; j++ { + idx.Fanout[j] = uint32(len(w.objects)) + } + + idx.Version = VersionSupported + idx.PackfileChecksum = w.checksum + + return idx, nil +} + +func (w *Writer) addOffset64(pos uint64) uint64 { + buf := new(bytes.Buffer) + binary.WriteUint64(buf, pos) + w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) + + index := uint64(w.offset64 | (1 << 31)) + w.offset64++ + + return index +} + +func (o objects) Len() int { + return len(o) +} + +func (o objects) Less(i int, j int) bool { + cmp := bytes.Compare(o[i].Hash[:], o[j].Hash[:]) + return cmp < 0 +} + +func (o objects) Swap(i int, j int) { + o[i], o[j] = o[j], o[i] +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/index/decoder.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/index/decoder.go index 1a58128f..df25530c 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/index/decoder.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/index/decoder.go @@ -21,7 +21,7 @@ var ( // ErrMalformedSignature is returned by Decode when the index header file is // malformed ErrMalformedSignature = errors.New("malformed index signature file") - // ErrInvalidChecksum is returned by Decode if the SHA1 hash missmatch with + // ErrInvalidChecksum is returned by Decode if the SHA1 hash mismatch with // the read content ErrInvalidChecksum = errors.New("invalid checksum") diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/common.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/common.go index beb015d3..2b4acebd 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/common.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/common.go @@ -23,24 +23,28 @@ const ( maskType = uint8(112) // 0111 0000 ) -// UpdateObjectStorage updates the given storer.EncodedObjectStorer with the contents of the +// UpdateObjectStorage updates the storer with the objects in the given // packfile. -func UpdateObjectStorage(s storer.EncodedObjectStorer, packfile io.Reader) error { - if sw, ok := s.(storer.PackfileWriter); ok { - return writePackfileToObjectStorage(sw, packfile) +func UpdateObjectStorage(s storer.Storer, packfile io.Reader) error { + if pw, ok := s.(storer.PackfileWriter); ok { + return WritePackfileToObjectStorage(pw, packfile) } - stream := NewScanner(packfile) - d, err := NewDecoder(stream, s) + p, err := NewParserWithStorage(NewScanner(packfile), s) if err != nil { return err } - _, err = d.Decode() + _, err = p.Parse() return err } -func writePackfileToObjectStorage(sw storer.PackfileWriter, packfile io.Reader) (err error) { +// WritePackfileToObjectStorage writes all the packfile objects into the given +// object storage. +func WritePackfileToObjectStorage( + sw storer.PackfileWriter, + packfile io.Reader, +) (err error) { w, err := sw.PackfileWriter() if err != nil { return err diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/decoder.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/decoder.go deleted file mode 100644 index f706e5d8..00000000 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/decoder.go +++ /dev/null @@ -1,495 +0,0 @@ -package packfile - -import ( - "bytes" - - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/cache" - "gopkg.in/src-d/go-git.v4/plumbing/storer" -) - -// Format specifies if the packfile uses ref-deltas or ofs-deltas. -type Format int - -// Possible values of the Format type. -const ( - UnknownFormat Format = iota - OFSDeltaFormat - REFDeltaFormat -) - -var ( - // ErrMaxObjectsLimitReached is returned by Decode when the number - // of objects in the packfile is higher than - // Decoder.MaxObjectsLimit. - ErrMaxObjectsLimitReached = NewError("max. objects limit reached") - // ErrInvalidObject is returned by Decode when an invalid object is - // found in the packfile. - ErrInvalidObject = NewError("invalid git object") - // ErrPackEntryNotFound is returned by Decode when a reference in - // the packfile references and unknown object. - ErrPackEntryNotFound = NewError("can't find a pack entry") - // ErrZLib is returned by Decode when there was an error unzipping - // the packfile contents. - ErrZLib = NewError("zlib reading error") - // ErrCannotRecall is returned by RecallByOffset or RecallByHash if the object - // to recall cannot be returned. - ErrCannotRecall = NewError("cannot recall object") - // ErrResolveDeltasNotSupported is returned if a NewDecoder is used with a - // non-seekable scanner and without a plumbing.ObjectStorage - ErrResolveDeltasNotSupported = NewError("resolve delta is not supported") - // ErrNonSeekable is returned if a ReadObjectAt method is called without a - // seekable scanner - ErrNonSeekable = NewError("non-seekable scanner") - // ErrRollback error making Rollback over a transaction after an error - ErrRollback = NewError("rollback error, during set error") - // ErrAlreadyDecoded is returned if NewDecoder is called for a second time - ErrAlreadyDecoded = NewError("packfile was already decoded") -) - -// Decoder reads and decodes packfiles from an input Scanner, if an ObjectStorer -// was provided the decoded objects are store there. If not the decode object -// is destroyed. The Offsets and CRCs are calculated whether an -// ObjectStorer was provided or not. -type Decoder struct { - deltaBaseCache cache.Object - - s *Scanner - o storer.EncodedObjectStorer - tx storer.Transaction - - isDecoded bool - - // hasBuiltIndex indicates if the index is fully built or not. If it is not, - // will be built incrementally while decoding. - hasBuiltIndex bool - idx *Index - - offsetToType map[int64]plumbing.ObjectType - decoderType plumbing.ObjectType -} - -// NewDecoder returns a new Decoder that decodes a Packfile using the given -// Scanner and stores the objects in the provided EncodedObjectStorer. ObjectStorer can be nil, in this -// If the passed EncodedObjectStorer is nil, objects are not stored, but -// offsets on the Packfile and CRCs are calculated. -// -// If EncodedObjectStorer is nil and the Scanner is not Seekable, ErrNonSeekable is -// returned. -// -// If the ObjectStorer implements storer.Transactioner, a transaction is created -// during the Decode execution. If anything fails, Rollback is called -func NewDecoder(s *Scanner, o storer.EncodedObjectStorer) (*Decoder, error) { - return NewDecoderForType(s, o, plumbing.AnyObject, - cache.NewObjectLRUDefault()) -} - -// NewDecoderWithCache is a version of NewDecoder where cache can be specified. -func NewDecoderWithCache(s *Scanner, o storer.EncodedObjectStorer, - cacheObject cache.Object) (*Decoder, error) { - - return NewDecoderForType(s, o, plumbing.AnyObject, cacheObject) -} - -// NewDecoderForType returns a new Decoder but in this case for a specific object type. -// When an object is read using this Decoder instance and it is not of the same type of -// the specified one, nil will be returned. This is intended to avoid the content -// deserialization of all the objects. -// -// cacheObject is a cache.Object implementation that is used to speed up the -// process. If cache is not needed you can pass nil. To create an LRU cache -// object with the default size you can use the helper cache.ObjectLRUDefault(). -func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer, - t plumbing.ObjectType, cacheObject cache.Object) (*Decoder, error) { - - if t == plumbing.OFSDeltaObject || - t == plumbing.REFDeltaObject || - t == plumbing.InvalidObject { - return nil, plumbing.ErrInvalidType - } - - if !canResolveDeltas(s, o) { - return nil, ErrResolveDeltasNotSupported - } - - return &Decoder{ - s: s, - o: o, - deltaBaseCache: cacheObject, - - idx: NewIndex(0), - offsetToType: make(map[int64]plumbing.ObjectType), - decoderType: t, - }, nil -} - -func canResolveDeltas(s *Scanner, o storer.EncodedObjectStorer) bool { - return s.IsSeekable || o != nil -} - -// Decode reads a packfile and stores it in the value pointed to by s. The -// offsets and the CRCs are calculated by this method -func (d *Decoder) Decode() (checksum plumbing.Hash, err error) { - defer func() { d.isDecoded = true }() - - if d.isDecoded { - return plumbing.ZeroHash, ErrAlreadyDecoded - } - - if err := d.doDecode(); err != nil { - return plumbing.ZeroHash, err - } - - return d.s.Checksum() -} - -func (d *Decoder) doDecode() error { - _, count, err := d.s.Header() - if err != nil { - return err - } - - if !d.hasBuiltIndex { - d.idx = NewIndex(int(count)) - } - defer func() { d.hasBuiltIndex = true }() - - _, isTxStorer := d.o.(storer.Transactioner) - switch { - case d.o == nil: - return d.decodeObjects(int(count)) - case isTxStorer: - return d.decodeObjectsWithObjectStorerTx(int(count)) - default: - return d.decodeObjectsWithObjectStorer(int(count)) - } -} - -func (d *Decoder) decodeObjects(count int) error { - for i := 0; i < count; i++ { - if _, err := d.DecodeObject(); err != nil { - return err - } - } - - return nil -} - -func (d *Decoder) decodeObjectsWithObjectStorer(count int) error { - for i := 0; i < count; i++ { - obj, err := d.DecodeObject() - if err != nil { - return err - } - - if _, err := d.o.SetEncodedObject(obj); err != nil { - return err - } - } - - return nil -} - -func (d *Decoder) decodeObjectsWithObjectStorerTx(count int) error { - d.tx = d.o.(storer.Transactioner).Begin() - - for i := 0; i < count; i++ { - obj, err := d.DecodeObject() - if err != nil { - return err - } - - if _, err := d.tx.SetEncodedObject(obj); err != nil { - if rerr := d.tx.Rollback(); rerr != nil { - return ErrRollback.AddDetails( - "error: %s, during tx.Set error: %s", rerr, err, - ) - } - - return err - } - - } - - return d.tx.Commit() -} - -// DecodeObject reads the next object from the scanner and returns it. This -// method can be used in replacement of the Decode method, to work in a -// interactive way. If you created a new decoder instance using NewDecoderForType -// constructor, if the object decoded is not equals to the specified one, nil will -// be returned -func (d *Decoder) DecodeObject() (plumbing.EncodedObject, error) { - return d.doDecodeObject(d.decoderType) -} - -func (d *Decoder) doDecodeObject(t plumbing.ObjectType) (plumbing.EncodedObject, error) { - h, err := d.s.NextObjectHeader() - if err != nil { - return nil, err - } - - if t == plumbing.AnyObject { - return d.decodeByHeader(h) - } - - return d.decodeIfSpecificType(h) -} - -func (d *Decoder) decodeIfSpecificType(h *ObjectHeader) (plumbing.EncodedObject, error) { - var ( - obj plumbing.EncodedObject - realType plumbing.ObjectType - err error - ) - switch h.Type { - case plumbing.OFSDeltaObject: - realType, err = d.ofsDeltaType(h.OffsetReference) - case plumbing.REFDeltaObject: - realType, err = d.refDeltaType(h.Reference) - if err == plumbing.ErrObjectNotFound { - obj, err = d.decodeByHeader(h) - if err != nil { - realType = obj.Type() - } - } - default: - realType = h.Type - } - - if err != nil { - return nil, err - } - - d.offsetToType[h.Offset] = realType - - if d.decoderType == realType { - if obj != nil { - return obj, nil - } - - return d.decodeByHeader(h) - } - - return nil, nil -} - -func (d *Decoder) ofsDeltaType(offset int64) (plumbing.ObjectType, error) { - t, ok := d.offsetToType[offset] - if !ok { - return plumbing.InvalidObject, plumbing.ErrObjectNotFound - } - - return t, nil -} - -func (d *Decoder) refDeltaType(ref plumbing.Hash) (plumbing.ObjectType, error) { - e, ok := d.idx.LookupHash(ref) - if !ok { - return plumbing.InvalidObject, plumbing.ErrObjectNotFound - } - - return d.ofsDeltaType(int64(e.Offset)) -} - -func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error) { - obj := d.newObject() - obj.SetSize(h.Length) - obj.SetType(h.Type) - - var crc uint32 - var err error - switch h.Type { - case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: - crc, err = d.fillRegularObjectContent(obj) - case plumbing.REFDeltaObject: - crc, err = d.fillREFDeltaObjectContent(obj, h.Reference) - case plumbing.OFSDeltaObject: - crc, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference) - default: - err = ErrInvalidObject.AddDetails("type %q", h.Type) - } - - if err != nil { - return obj, err - } - - if !d.hasBuiltIndex { - d.idx.Add(obj.Hash(), uint64(h.Offset), crc) - } - - return obj, nil -} - -func (d *Decoder) newObject() plumbing.EncodedObject { - if d.o == nil { - return &plumbing.MemoryObject{} - } - - return d.o.NewEncodedObject() -} - -// DecodeObjectAt reads an object at the given location. Every EncodedObject -// returned is added into a internal index. This is intended to be able to regenerate -// objects from deltas (offset deltas or reference deltas) without an package index -// (.idx file). If Decode wasn't called previously objects offset should provided -// using the SetOffsets method. It decodes the object regardless of the Decoder -// type. -func (d *Decoder) DecodeObjectAt(offset int64) (plumbing.EncodedObject, error) { - if !d.s.IsSeekable { - return nil, ErrNonSeekable - } - - beforeJump, err := d.s.SeekFromStart(offset) - if err != nil { - return nil, err - } - - defer func() { - _, seekErr := d.s.SeekFromStart(beforeJump) - if err == nil { - err = seekErr - } - }() - - return d.doDecodeObject(plumbing.AnyObject) -} - -func (d *Decoder) fillRegularObjectContent(obj plumbing.EncodedObject) (uint32, error) { - w, err := obj.Writer() - if err != nil { - return 0, err - } - - _, crc, err := d.s.NextObject(w) - return crc, err -} - -func (d *Decoder) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) (uint32, error) { - buf := bufPool.Get().(*bytes.Buffer) - buf.Reset() - _, crc, err := d.s.NextObject(buf) - if err != nil { - return 0, err - } - - base, ok := d.cacheGet(ref) - if !ok { - base, err = d.recallByHash(ref) - if err != nil { - return 0, err - } - } - - obj.SetType(base.Type()) - err = ApplyDelta(obj, base, buf.Bytes()) - d.cachePut(obj) - bufPool.Put(buf) - - return crc, err -} - -func (d *Decoder) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset int64) (uint32, error) { - buf := bytes.NewBuffer(nil) - _, crc, err := d.s.NextObject(buf) - if err != nil { - return 0, err - } - - e, ok := d.idx.LookupOffset(uint64(offset)) - var base plumbing.EncodedObject - if ok { - base, ok = d.cacheGet(e.Hash) - } - - if !ok { - base, err = d.recallByOffset(offset) - if err != nil { - return 0, err - } - - d.cachePut(base) - } - - obj.SetType(base.Type()) - err = ApplyDelta(obj, base, buf.Bytes()) - d.cachePut(obj) - - return crc, err -} - -func (d *Decoder) cacheGet(h plumbing.Hash) (plumbing.EncodedObject, bool) { - if d.deltaBaseCache == nil { - return nil, false - } - - return d.deltaBaseCache.Get(h) -} - -func (d *Decoder) cachePut(obj plumbing.EncodedObject) { - if d.deltaBaseCache == nil { - return - } - - d.deltaBaseCache.Put(obj) -} - -func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { - if d.s.IsSeekable { - return d.DecodeObjectAt(o) - } - - if e, ok := d.idx.LookupOffset(uint64(o)); ok { - return d.recallByHashNonSeekable(e.Hash) - } - - return nil, plumbing.ErrObjectNotFound -} - -func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) { - if d.s.IsSeekable { - if e, ok := d.idx.LookupHash(h); ok { - return d.DecodeObjectAt(int64(e.Offset)) - } - } - - return d.recallByHashNonSeekable(h) -} - -// recallByHashNonSeekable if we are in a transaction the objects are read from -// the transaction, if not are directly read from the ObjectStorer -func (d *Decoder) recallByHashNonSeekable(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { - if d.tx != nil { - obj, err = d.tx.EncodedObject(plumbing.AnyObject, h) - } else { - obj, err = d.o.EncodedObject(plumbing.AnyObject, h) - } - - if err != plumbing.ErrObjectNotFound { - return obj, err - } - - return nil, plumbing.ErrObjectNotFound -} - -// SetIndex sets an index for the packfile. It is recommended to set this. -// The index might be read from a file or reused from a previous Decoder usage -// (see Index function). -func (d *Decoder) SetIndex(idx *Index) { - d.hasBuiltIndex = true - d.idx = idx -} - -// Index returns the index for the packfile. If index was set with SetIndex, -// Index will return it. Otherwise, it will return an index that is built while -// decoding. If neither SetIndex was called with a full index or Decode called -// for the whole packfile, then the returned index will be incomplete. -func (d *Decoder) Index() *Index { - return d.idx -} - -// Close closes the Scanner. usually this mean that the whole reader is read and -// discarded -func (d *Decoder) Close() error { - return d.s.Close() -} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/fsobject.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/fsobject.go new file mode 100644 index 00000000..330cb73c --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/fsobject.go @@ -0,0 +1,116 @@ +package packfile + +import ( + "io" + + billy "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" +) + +// FSObject is an object from the packfile on the filesystem. +type FSObject struct { + hash plumbing.Hash + h *ObjectHeader + offset int64 + size int64 + typ plumbing.ObjectType + index idxfile.Index + fs billy.Filesystem + path string + cache cache.Object +} + +// NewFSObject creates a new filesystem object. +func NewFSObject( + hash plumbing.Hash, + finalType plumbing.ObjectType, + offset int64, + contentSize int64, + index idxfile.Index, + fs billy.Filesystem, + path string, + cache cache.Object, +) *FSObject { + return &FSObject{ + hash: hash, + offset: offset, + size: contentSize, + typ: finalType, + index: index, + fs: fs, + path: path, + cache: cache, + } +} + +// Reader implements the plumbing.EncodedObject interface. +func (o *FSObject) Reader() (io.ReadCloser, error) { + obj, ok := o.cache.Get(o.hash) + if ok { + reader, err := obj.Reader() + if err != nil { + return nil, err + } + + return reader, nil + } + + f, err := o.fs.Open(o.path) + if err != nil { + return nil, err + } + + p := NewPackfileWithCache(o.index, nil, f, o.cache) + r, err := p.getObjectContent(o.offset) + if err != nil { + _ = f.Close() + return nil, err + } + + if err := f.Close(); err != nil { + return nil, err + } + + return r, nil +} + +// SetSize implements the plumbing.EncodedObject interface. This method +// is a noop. +func (o *FSObject) SetSize(int64) {} + +// SetType implements the plumbing.EncodedObject interface. This method is +// a noop. +func (o *FSObject) SetType(plumbing.ObjectType) {} + +// Hash implements the plumbing.EncodedObject interface. +func (o *FSObject) Hash() plumbing.Hash { return o.hash } + +// Size implements the plumbing.EncodedObject interface. +func (o *FSObject) Size() int64 { return o.size } + +// Type implements the plumbing.EncodedObject interface. +func (o *FSObject) Type() plumbing.ObjectType { + return o.typ +} + +// Writer implements the plumbing.EncodedObject interface. This method always +// returns a nil writer. +func (o *FSObject) Writer() (io.WriteCloser, error) { + return nil, nil +} + +type objectReader struct { + io.ReadCloser + f billy.File +} + +func (r *objectReader) Close() error { + if err := r.ReadCloser.Close(); err != nil { + _ = r.f.Close() + return err + } + + return r.f.Close() +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/index.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/index.go deleted file mode 100644 index 7d8f2ad1..00000000 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/index.go +++ /dev/null @@ -1,115 +0,0 @@ -package packfile - -import ( - "sort" - - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" -) - -// Index is an in-memory representation of a packfile index. -// This uses idxfile.Idxfile under the hood to obtain indexes from .idx files -// or to store them. -type Index struct { - byHash map[plumbing.Hash]*idxfile.Entry - byOffset []*idxfile.Entry // sorted by their offset -} - -// NewIndex creates a new empty index with the given size. Size is a hint and -// can be 0. It is recommended to set it to the number of objects to be indexed -// if it is known beforehand (e.g. reading from a packfile). -func NewIndex(size int) *Index { - return &Index{ - byHash: make(map[plumbing.Hash]*idxfile.Entry, size), - byOffset: make([]*idxfile.Entry, 0, size), - } -} - -// NewIndexFromIdxFile creates a new Index from an idxfile.IdxFile. -func NewIndexFromIdxFile(idxf *idxfile.Idxfile) *Index { - idx := &Index{ - byHash: make(map[plumbing.Hash]*idxfile.Entry, idxf.ObjectCount), - byOffset: make([]*idxfile.Entry, 0, idxf.ObjectCount), - } - for _, e := range idxf.Entries { - idx.addUnsorted(e) - } - sort.Sort(orderByOffset(idx.byOffset)) - - return idx -} - -// orderByOffset is a sort.Interface adapter that arranges -// a slice of entries by their offset. -type orderByOffset []*idxfile.Entry - -func (o orderByOffset) Len() int { return len(o) } -func (o orderByOffset) Less(i, j int) bool { return o[i].Offset < o[j].Offset } -func (o orderByOffset) Swap(i, j int) { o[i], o[j] = o[j], o[i] } - -// Add adds a new Entry with the given values to the index. -func (idx *Index) Add(h plumbing.Hash, offset uint64, crc32 uint32) { - e := &idxfile.Entry{ - Hash: h, - Offset: offset, - CRC32: crc32, - } - idx.byHash[e.Hash] = e - - // Find the right position in byOffset. - // Look for the first position whose offset is *greater* than e.Offset. - i := sort.Search(len(idx.byOffset), func(i int) bool { - return idx.byOffset[i].Offset > offset - }) - if i == len(idx.byOffset) { - // Simple case: add it to the end. - idx.byOffset = append(idx.byOffset, e) - return - } - // Harder case: shift existing entries down by one to make room. - // Append a nil entry first so we can use existing capacity in case - // the index was carefully preallocated. - idx.byOffset = append(idx.byOffset, nil) - copy(idx.byOffset[i+1:], idx.byOffset[i:len(idx.byOffset)-1]) - idx.byOffset[i] = e -} - -func (idx *Index) addUnsorted(e *idxfile.Entry) { - idx.byHash[e.Hash] = e - idx.byOffset = append(idx.byOffset, e) -} - -// LookupHash looks an entry up by its hash. An idxfile.Entry is returned and -// a bool, which is true if it was found or false if it wasn't. -func (idx *Index) LookupHash(h plumbing.Hash) (*idxfile.Entry, bool) { - e, ok := idx.byHash[h] - return e, ok -} - -// LookupHash looks an entry up by its offset in the packfile. An idxfile.Entry -// is returned and a bool, which is true if it was found or false if it wasn't. -func (idx *Index) LookupOffset(offset uint64) (*idxfile.Entry, bool) { - i := sort.Search(len(idx.byOffset), func(i int) bool { - return idx.byOffset[i].Offset >= offset - }) - if i >= len(idx.byOffset) || idx.byOffset[i].Offset != offset { - return nil, false // not present - } - return idx.byOffset[i], true -} - -// Size returns the number of entries in the index. -func (idx *Index) Size() int { - return len(idx.byHash) -} - -// ToIdxFile converts the index to an idxfile.Idxfile, which can then be used -// to serialize. -func (idx *Index) ToIdxFile() *idxfile.Idxfile { - idxf := idxfile.NewIdxfile() - for _, e := range idx.byHash { - idxf.Entries = append(idxf.Entries, e) - } - - return idxf -} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/packfile.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/packfile.go new file mode 100644 index 00000000..0d13066b --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/packfile.go @@ -0,0 +1,538 @@ +package packfile + +import ( + "bytes" + "io" + "os" + + billy "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" + "gopkg.in/src-d/go-git.v4/plumbing/storer" +) + +var ( + // ErrInvalidObject is returned by Decode when an invalid object is + // found in the packfile. + ErrInvalidObject = NewError("invalid git object") + // ErrZLib is returned by Decode when there was an error unzipping + // the packfile contents. + ErrZLib = NewError("zlib reading error") +) + +// Packfile allows retrieving information from inside a packfile. +type Packfile struct { + idxfile.Index + fs billy.Filesystem + file billy.File + s *Scanner + deltaBaseCache cache.Object + offsetToType map[int64]plumbing.ObjectType +} + +// NewPackfileWithCache creates a new Packfile with the given object cache. +// If the filesystem is provided, the packfile will return FSObjects, otherwise +// it will return MemoryObjects. +func NewPackfileWithCache( + index idxfile.Index, + fs billy.Filesystem, + file billy.File, + cache cache.Object, +) *Packfile { + s := NewScanner(file) + return &Packfile{ + index, + fs, + file, + s, + cache, + make(map[int64]plumbing.ObjectType), + } +} + +// NewPackfile returns a packfile representation for the given packfile file +// and packfile idx. +// If the filesystem is provided, the packfile will return FSObjects, otherwise +// it will return MemoryObjects. +func NewPackfile(index idxfile.Index, fs billy.Filesystem, file billy.File) *Packfile { + return NewPackfileWithCache(index, fs, file, cache.NewObjectLRUDefault()) +} + +// Get retrieves the encoded object in the packfile with the given hash. +func (p *Packfile) Get(h plumbing.Hash) (plumbing.EncodedObject, error) { + offset, err := p.FindOffset(h) + if err != nil { + return nil, err + } + + return p.GetByOffset(offset) +} + +// GetByOffset retrieves the encoded object from the packfile with the given +// offset. +func (p *Packfile) GetByOffset(o int64) (plumbing.EncodedObject, error) { + hash, err := p.FindHash(o) + if err == nil { + if obj, ok := p.deltaBaseCache.Get(hash); ok { + return obj, nil + } + } + + if _, err := p.s.SeekFromStart(o); err != nil { + if err == io.EOF || isInvalid(err) { + return nil, plumbing.ErrObjectNotFound + } + + return nil, err + } + + return p.nextObject() +} + +// GetSizeByOffset retrieves the size of the encoded object from the +// packfile with the given offset. +func (p *Packfile) GetSizeByOffset(o int64) (size int64, err error) { + if _, err := p.s.SeekFromStart(o); err != nil { + if err == io.EOF || isInvalid(err) { + return 0, plumbing.ErrObjectNotFound + } + + return 0, err + } + + h, err := p.nextObjectHeader() + if err != nil { + return 0, err + } + return h.Length, nil +} + +func (p *Packfile) nextObjectHeader() (*ObjectHeader, error) { + h, err := p.s.NextObjectHeader() + p.s.pendingObject = nil + return h, err +} + +func (p *Packfile) getObjectData( + h *ObjectHeader, +) (typ plumbing.ObjectType, size int64, err error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + typ = h.Type + size = h.Length + case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + + _, _, err = p.s.NextObject(buf) + if err != nil { + return + } + + delta := buf.Bytes() + _, delta = decodeLEB128(delta) // skip src size + sz, _ := decodeLEB128(delta) + size = int64(sz) + + var offset int64 + if h.Type == plumbing.REFDeltaObject { + offset, err = p.FindOffset(h.Reference) + if err != nil { + return + } + } else { + offset = h.OffsetReference + } + + if baseType, ok := p.offsetToType[offset]; ok { + typ = baseType + } else { + if _, err = p.s.SeekFromStart(offset); err != nil { + return + } + + h, err = p.nextObjectHeader() + if err != nil { + return + } + + typ, _, err = p.getObjectData(h) + if err != nil { + return + } + } + default: + err = ErrInvalidObject.AddDetails("type %q", h.Type) + } + + return +} + +func (p *Packfile) getObjectSize(h *ObjectHeader) (int64, error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + return h.Length, nil + case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + + if _, _, err := p.s.NextObject(buf); err != nil { + return 0, err + } + + delta := buf.Bytes() + _, delta = decodeLEB128(delta) // skip src size + sz, _ := decodeLEB128(delta) + return int64(sz), nil + default: + return 0, ErrInvalidObject.AddDetails("type %q", h.Type) + } +} + +func (p *Packfile) getObjectType(h *ObjectHeader) (typ plumbing.ObjectType, err error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + return h.Type, nil + case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: + var offset int64 + if h.Type == plumbing.REFDeltaObject { + offset, err = p.FindOffset(h.Reference) + if err != nil { + return + } + } else { + offset = h.OffsetReference + } + + if baseType, ok := p.offsetToType[offset]; ok { + typ = baseType + } else { + if _, err = p.s.SeekFromStart(offset); err != nil { + return + } + + h, err = p.nextObjectHeader() + if err != nil { + return + } + + typ, err = p.getObjectType(h) + if err != nil { + return + } + } + default: + err = ErrInvalidObject.AddDetails("type %q", h.Type) + } + + return +} + +func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { + h, err := p.nextObjectHeader() + if err != nil { + if err == io.EOF || isInvalid(err) { + return nil, plumbing.ErrObjectNotFound + } + return nil, err + } + + // If we have no filesystem, we will return a MemoryObject instead + // of an FSObject. + if p.fs == nil { + return p.getNextObject(h) + } + + hash, err := p.FindHash(h.Offset) + if err != nil { + return nil, err + } + + size, err := p.getObjectSize(h) + if err != nil { + return nil, err + } + + typ, err := p.getObjectType(h) + if err != nil { + return nil, err + } + + p.offsetToType[h.Offset] = typ + + return NewFSObject( + hash, + typ, + h.Offset, + size, + p.Index, + p.fs, + p.file.Name(), + p.deltaBaseCache, + ), nil +} + +func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { + ref, err := p.FindHash(offset) + if err == nil { + obj, ok := p.cacheGet(ref) + if ok { + reader, err := obj.Reader() + if err != nil { + return nil, err + } + + return reader, nil + } + } + + if _, err := p.s.SeekFromStart(offset); err != nil { + return nil, err + } + + h, err := p.nextObjectHeader() + if err != nil { + return nil, err + } + + obj, err := p.getNextObject(h) + if err != nil { + return nil, err + } + + return obj.Reader() +} + +func (p *Packfile) getNextObject(h *ObjectHeader) (plumbing.EncodedObject, error) { + var obj = new(plumbing.MemoryObject) + obj.SetSize(h.Length) + obj.SetType(h.Type) + + var err error + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + err = p.fillRegularObjectContent(obj) + case plumbing.REFDeltaObject: + err = p.fillREFDeltaObjectContent(obj, h.Reference) + case plumbing.OFSDeltaObject: + err = p.fillOFSDeltaObjectContent(obj, h.OffsetReference) + default: + err = ErrInvalidObject.AddDetails("type %q", h.Type) + } + + if err != nil { + return nil, err + } + + return obj, nil +} + +func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error { + w, err := obj.Writer() + if err != nil { + return err + } + + _, _, err = p.s.NextObject(w) + p.cachePut(obj) + + return err +} + +func (p *Packfile) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) error { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + _, _, err := p.s.NextObject(buf) + if err != nil { + return err + } + + base, ok := p.cacheGet(ref) + if !ok { + base, err = p.Get(ref) + if err != nil { + return err + } + } + + obj.SetType(base.Type()) + err = ApplyDelta(obj, base, buf.Bytes()) + p.cachePut(obj) + bufPool.Put(buf) + + return err +} + +func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset int64) error { + buf := bytes.NewBuffer(nil) + _, _, err := p.s.NextObject(buf) + if err != nil { + return err + } + + var base plumbing.EncodedObject + var ok bool + hash, err := p.FindHash(offset) + if err == nil { + base, ok = p.cacheGet(hash) + } + + if !ok { + base, err = p.GetByOffset(offset) + if err != nil { + return err + } + + p.cachePut(base) + } + + obj.SetType(base.Type()) + err = ApplyDelta(obj, base, buf.Bytes()) + p.cachePut(obj) + + return err +} + +func (p *Packfile) cacheGet(h plumbing.Hash) (plumbing.EncodedObject, bool) { + if p.deltaBaseCache == nil { + return nil, false + } + + return p.deltaBaseCache.Get(h) +} + +func (p *Packfile) cachePut(obj plumbing.EncodedObject) { + if p.deltaBaseCache == nil { + return + } + + p.deltaBaseCache.Put(obj) +} + +// GetAll returns an iterator with all encoded objects in the packfile. +// The iterator returned is not thread-safe, it should be used in the same +// thread as the Packfile instance. +func (p *Packfile) GetAll() (storer.EncodedObjectIter, error) { + return p.GetByType(plumbing.AnyObject) +} + +// GetByType returns all the objects of the given type. +func (p *Packfile) GetByType(typ plumbing.ObjectType) (storer.EncodedObjectIter, error) { + switch typ { + case plumbing.AnyObject, + plumbing.BlobObject, + plumbing.TreeObject, + plumbing.CommitObject, + plumbing.TagObject: + entries, err := p.EntriesByOffset() + if err != nil { + return nil, err + } + + return &objectIter{ + // Easiest way to provide an object decoder is just to pass a Packfile + // instance. To not mess with the seeks, it's a new instance with a + // different scanner but the same cache and offset to hash map for + // reusing as much cache as possible. + p: p, + iter: entries, + typ: typ, + }, nil + default: + return nil, plumbing.ErrInvalidType + } +} + +// ID returns the ID of the packfile, which is the checksum at the end of it. +func (p *Packfile) ID() (plumbing.Hash, error) { + prev, err := p.file.Seek(-20, io.SeekEnd) + if err != nil { + return plumbing.ZeroHash, err + } + + var hash plumbing.Hash + if _, err := io.ReadFull(p.file, hash[:]); err != nil { + return plumbing.ZeroHash, err + } + + if _, err := p.file.Seek(prev, io.SeekStart); err != nil { + return plumbing.ZeroHash, err + } + + return hash, nil +} + +// Close the packfile and its resources. +func (p *Packfile) Close() error { + closer, ok := p.file.(io.Closer) + if !ok { + return nil + } + + return closer.Close() +} + +type objectIter struct { + p *Packfile + typ plumbing.ObjectType + iter idxfile.EntryIter +} + +func (i *objectIter) Next() (plumbing.EncodedObject, error) { + for { + e, err := i.iter.Next() + if err != nil { + return nil, err + } + + obj, err := i.p.GetByOffset(int64(e.Offset)) + if err != nil { + return nil, err + } + + if i.typ == plumbing.AnyObject || obj.Type() == i.typ { + return obj, nil + } + } +} + +func (i *objectIter) ForEach(f func(plumbing.EncodedObject) error) error { + for { + o, err := i.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := f(o); err != nil { + return err + } + } +} + +func (i *objectIter) Close() { + i.iter.Close() +} + +// isInvalid checks whether an error is an os.PathError with an os.ErrInvalid +// error inside. It also checks for the windows error, which is different from +// os.ErrInvalid. +func isInvalid(err error) bool { + pe, ok := err.(*os.PathError) + if !ok { + return false + } + + errstr := pe.Err.Error() + return errstr == errInvalidUnix || errstr == errInvalidWindows +} + +// errInvalidWindows is the Windows equivalent to os.ErrInvalid +const errInvalidWindows = "The parameter is incorrect." + +var errInvalidUnix = os.ErrInvalid.Error() diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/parser.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/parser.go new file mode 100644 index 00000000..28582b5f --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/parser.go @@ -0,0 +1,489 @@ +package packfile + +import ( + "bytes" + "errors" + "io" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/storer" +) + +var ( + // ErrReferenceDeltaNotFound is returned when the reference delta is not + // found. + ErrReferenceDeltaNotFound = errors.New("reference delta not found") + + // ErrNotSeekableSource is returned when the source for the parser is not + // seekable and a storage was not provided, so it can't be parsed. + ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided") + + // ErrDeltaNotCached is returned when the delta could not be found in cache. + ErrDeltaNotCached = errors.New("delta could not be found in cache") +) + +// Observer interface is implemented by index encoders. +type Observer interface { + // OnHeader is called when a new packfile is opened. + OnHeader(count uint32) error + // OnInflatedObjectHeader is called for each object header read. + OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error + // OnInflatedObjectContent is called for each decoded object. + OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, content []byte) error + // OnFooter is called when decoding is done. + OnFooter(h plumbing.Hash) error +} + +// Parser decodes a packfile and calls any observer associated to it. Is used +// to generate indexes. +type Parser struct { + storage storer.EncodedObjectStorer + scanner *Scanner + count uint32 + oi []*objectInfo + oiByHash map[plumbing.Hash]*objectInfo + oiByOffset map[int64]*objectInfo + hashOffset map[plumbing.Hash]int64 + pendingRefDeltas map[plumbing.Hash][]*objectInfo + checksum plumbing.Hash + + cache *cache.BufferLRU + // delta content by offset, only used if source is not seekable + deltas map[int64][]byte + + ob []Observer +} + +// NewParser creates a new Parser. The Scanner source must be seekable. +// If it's not, NewParserWithStorage should be used instead. +func NewParser(scanner *Scanner, ob ...Observer) (*Parser, error) { + return NewParserWithStorage(scanner, nil, ob...) +} + +// NewParserWithStorage creates a new Parser. The scanner source must either +// be seekable or a storage must be provided. +func NewParserWithStorage( + scanner *Scanner, + storage storer.EncodedObjectStorer, + ob ...Observer, +) (*Parser, error) { + if !scanner.IsSeekable && storage == nil { + return nil, ErrNotSeekableSource + } + + var deltas map[int64][]byte + if !scanner.IsSeekable { + deltas = make(map[int64][]byte) + } + + return &Parser{ + storage: storage, + scanner: scanner, + ob: ob, + count: 0, + cache: cache.NewBufferLRUDefault(), + pendingRefDeltas: make(map[plumbing.Hash][]*objectInfo), + deltas: deltas, + }, nil +} + +func (p *Parser) forEachObserver(f func(o Observer) error) error { + for _, o := range p.ob { + if err := f(o); err != nil { + return err + } + } + return nil +} + +func (p *Parser) onHeader(count uint32) error { + return p.forEachObserver(func(o Observer) error { + return o.OnHeader(count) + }) +} + +func (p *Parser) onInflatedObjectHeader( + t plumbing.ObjectType, + objSize int64, + pos int64, +) error { + return p.forEachObserver(func(o Observer) error { + return o.OnInflatedObjectHeader(t, objSize, pos) + }) +} + +func (p *Parser) onInflatedObjectContent( + h plumbing.Hash, + pos int64, + crc uint32, + content []byte, +) error { + return p.forEachObserver(func(o Observer) error { + return o.OnInflatedObjectContent(h, pos, crc, content) + }) +} + +func (p *Parser) onFooter(h plumbing.Hash) error { + return p.forEachObserver(func(o Observer) error { + return o.OnFooter(h) + }) +} + +// Parse start decoding phase of the packfile. +func (p *Parser) Parse() (plumbing.Hash, error) { + if err := p.init(); err != nil { + return plumbing.ZeroHash, err + } + + if err := p.indexObjects(); err != nil { + return plumbing.ZeroHash, err + } + + var err error + p.checksum, err = p.scanner.Checksum() + if err != nil && err != io.EOF { + return plumbing.ZeroHash, err + } + + if err := p.resolveDeltas(); err != nil { + return plumbing.ZeroHash, err + } + + if len(p.pendingRefDeltas) > 0 { + return plumbing.ZeroHash, ErrReferenceDeltaNotFound + } + + if err := p.onFooter(p.checksum); err != nil { + return plumbing.ZeroHash, err + } + + return p.checksum, nil +} + +func (p *Parser) init() error { + _, c, err := p.scanner.Header() + if err != nil { + return err + } + + if err := p.onHeader(c); err != nil { + return err + } + + p.count = c + p.oiByHash = make(map[plumbing.Hash]*objectInfo, p.count) + p.oiByOffset = make(map[int64]*objectInfo, p.count) + p.oi = make([]*objectInfo, p.count) + + return nil +} + +func (p *Parser) indexObjects() error { + buf := new(bytes.Buffer) + + for i := uint32(0); i < p.count; i++ { + buf.Reset() + + oh, err := p.scanner.NextObjectHeader() + if err != nil { + return err + } + + delta := false + var ota *objectInfo + switch t := oh.Type; t { + case plumbing.OFSDeltaObject: + delta = true + + parent, ok := p.oiByOffset[oh.OffsetReference] + if !ok { + return plumbing.ErrObjectNotFound + } + + ota = newDeltaObject(oh.Offset, oh.Length, t, parent) + parent.Children = append(parent.Children, ota) + case plumbing.REFDeltaObject: + delta = true + + parent, ok := p.oiByHash[oh.Reference] + if ok { + ota = newDeltaObject(oh.Offset, oh.Length, t, parent) + parent.Children = append(parent.Children, ota) + } else { + ota = newBaseObject(oh.Offset, oh.Length, t) + p.pendingRefDeltas[oh.Reference] = append( + p.pendingRefDeltas[oh.Reference], + ota, + ) + } + default: + ota = newBaseObject(oh.Offset, oh.Length, t) + } + + _, crc, err := p.scanner.NextObject(buf) + if err != nil { + return err + } + + ota.Crc32 = crc + ota.Length = oh.Length + + data := buf.Bytes() + if !delta { + sha1, err := getSHA1(ota.Type, data) + if err != nil { + return err + } + + ota.SHA1 = sha1 + p.oiByHash[ota.SHA1] = ota + } + + if p.storage != nil && !delta { + obj := new(plumbing.MemoryObject) + obj.SetSize(oh.Length) + obj.SetType(oh.Type) + if _, err := obj.Write(data); err != nil { + return err + } + + if _, err := p.storage.SetEncodedObject(obj); err != nil { + return err + } + } + + if delta && !p.scanner.IsSeekable { + p.deltas[oh.Offset] = make([]byte, len(data)) + copy(p.deltas[oh.Offset], data) + } + + p.oiByOffset[oh.Offset] = ota + p.oi[i] = ota + } + + return nil +} + +func (p *Parser) resolveDeltas() error { + for _, obj := range p.oi { + content, err := p.get(obj) + if err != nil { + return err + } + + if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil { + return err + } + + if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil { + return err + } + + if !obj.IsDelta() && len(obj.Children) > 0 { + for _, child := range obj.Children { + if _, err := p.resolveObject(child, content); err != nil { + return err + } + } + + // Remove the delta from the cache. + if obj.DiskType.IsDelta() && !p.scanner.IsSeekable { + delete(p.deltas, obj.Offset) + } + } + } + + return nil +} + +func (p *Parser) get(o *objectInfo) ([]byte, error) { + b, ok := p.cache.Get(o.Offset) + // If it's not on the cache and is not a delta we can try to find it in the + // storage, if there's one. + if !ok && p.storage != nil && !o.Type.IsDelta() { + var err error + e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1) + if err != nil { + return nil, err + } + + r, err := e.Reader() + if err != nil { + return nil, err + } + + b = make([]byte, e.Size()) + if _, err = r.Read(b); err != nil { + return nil, err + } + } + + if b != nil { + return b, nil + } + + var data []byte + if o.DiskType.IsDelta() { + base, err := p.get(o.Parent) + if err != nil { + return nil, err + } + + data, err = p.resolveObject(o, base) + if err != nil { + return nil, err + } + } else { + var err error + data, err = p.readData(o) + if err != nil { + return nil, err + } + } + + if len(o.Children) > 0 { + p.cache.Put(o.Offset, data) + } + + return data, nil +} + +func (p *Parser) resolveObject( + o *objectInfo, + base []byte, +) ([]byte, error) { + if !o.DiskType.IsDelta() { + return nil, nil + } + + data, err := p.readData(o) + if err != nil { + return nil, err + } + + data, err = applyPatchBase(o, data, base) + if err != nil { + return nil, err + } + + if pending, ok := p.pendingRefDeltas[o.SHA1]; ok { + for _, po := range pending { + po.Parent = o + o.Children = append(o.Children, po) + } + delete(p.pendingRefDeltas, o.SHA1) + } + + if p.storage != nil { + obj := new(plumbing.MemoryObject) + obj.SetSize(o.Size()) + obj.SetType(o.Type) + if _, err := obj.Write(data); err != nil { + return nil, err + } + + if _, err := p.storage.SetEncodedObject(obj); err != nil { + return nil, err + } + } + + return data, nil +} + +func (p *Parser) readData(o *objectInfo) ([]byte, error) { + if !p.scanner.IsSeekable && o.DiskType.IsDelta() { + data, ok := p.deltas[o.Offset] + if !ok { + return nil, ErrDeltaNotCached + } + + return data, nil + } + + if _, err := p.scanner.SeekFromStart(o.Offset); err != nil { + return nil, err + } + + if _, err := p.scanner.NextObjectHeader(); err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + if _, _, err := p.scanner.NextObject(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { + patched, err := PatchDelta(base, data) + if err != nil { + return nil, err + } + + if ota.SHA1 == plumbing.ZeroHash { + ota.Type = ota.Parent.Type + sha1, err := getSHA1(ota.Type, patched) + if err != nil { + return nil, err + } + + ota.SHA1 = sha1 + ota.Length = int64(len(patched)) + } + + return patched, nil +} + +func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) { + hasher := plumbing.NewHasher(t, int64(len(data))) + if _, err := hasher.Write(data); err != nil { + return plumbing.ZeroHash, err + } + + return hasher.Sum(), nil +} + +type objectInfo struct { + Offset int64 + Length int64 + Type plumbing.ObjectType + DiskType plumbing.ObjectType + + Crc32 uint32 + + Parent *objectInfo + Children []*objectInfo + SHA1 plumbing.Hash +} + +func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo { + return newDeltaObject(offset, length, t, nil) +} + +func newDeltaObject( + offset, length int64, + t plumbing.ObjectType, + parent *objectInfo, +) *objectInfo { + obj := &objectInfo{ + Offset: offset, + Length: length, + Type: t, + DiskType: t, + Crc32: 0, + Parent: parent, + } + + return obj +} + +func (o *objectInfo) IsDelta() bool { + return o.Type.IsDelta() +} + +func (o *objectInfo) Size() int64 { + return o.Length +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/patch_delta.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/patch_delta.go index c6048517..a972f1c4 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/patch_delta.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/patch_delta.go @@ -63,8 +63,8 @@ func PatchDelta(src, delta []byte) ([]byte, error) { targetSz, delta := decodeLEB128(delta) remainingTargetSz := targetSz - var dest []byte var cmd byte + dest := make([]byte, 0, targetSz) for { if len(delta) == 0 { return nil, ErrInvalidDelta diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/memory.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/memory.go index 51cbb54d..b8e1e1b8 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/memory.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/memory.go @@ -14,10 +14,10 @@ type MemoryObject struct { sz int64 } -// Hash return the object Hash, the hash is calculated on-the-fly the first -// time is called, the subsequent calls the same Hash is returned even if the -// type or the content has changed. The Hash is only generated if the size of -// the content is exactly the Object.Size +// Hash returns the object Hash, the hash is calculated on-the-fly the first +// time it's called, in all subsequent calls the same Hash is returned even +// if the type or the content have changed. The Hash is only generated if the +// size of the content is exactly the object size. func (o *MemoryObject) Hash() Hash { if o.h == ZeroHash && int64(len(o.cont)) == o.sz { o.h = ComputeHash(o.t, o.cont) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change.go index 729ff5a3..a1b4c274 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "fmt" "strings" @@ -81,7 +82,15 @@ func (c *Change) String() string { // Patch returns a Patch with all the file changes in chunks. This // representation can be used to create several diff outputs. func (c *Change) Patch() (*Patch, error) { - return getPatch("", c) + return c.PatchContext(context.Background()) +} + +// Patch returns a Patch with all the file changes in chunks. This +// representation can be used to create several diff outputs. +// If context expires, an non-nil error will be returned +// Provided context must be non-nil +func (c *Change) PatchContext(ctx context.Context) (*Patch, error) { + return getPatchContext(ctx, "", c) } func (c *Change) name() string { @@ -136,5 +145,13 @@ func (c Changes) String() string { // Patch returns a Patch with all the changes in chunks. This // representation can be used to create several diff outputs. func (c Changes) Patch() (*Patch, error) { - return getPatch("", c...) + return c.PatchContext(context.Background()) +} + +// Patch returns a Patch with all the changes in chunks. This +// representation can be used to create several diff outputs. +// If context expires, an non-nil error will be returned +// Provided context must be non-nil +func (c Changes) PatchContext(ctx context.Context) (*Patch, error) { + return getPatchContext(ctx, "", c...) } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit.go index c9a4c0ee..e2543426 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit.go @@ -3,6 +3,7 @@ package object import ( "bufio" "bytes" + "context" "errors" "fmt" "io" @@ -16,8 +17,9 @@ import ( ) const ( - beginpgp string = "-----BEGIN PGP SIGNATURE-----" - endpgp string = "-----END PGP SIGNATURE-----" + beginpgp string = "-----BEGIN PGP SIGNATURE-----" + endpgp string = "-----END PGP SIGNATURE-----" + headerpgp string = "gpgsig" ) // Hash represents the hash of an object @@ -75,7 +77,8 @@ func (c *Commit) Tree() (*Tree, error) { } // Patch returns the Patch between the actual commit and the provided one. -func (c *Commit) Patch(to *Commit) (*Patch, error) { +// Error will be return if context expires. Provided context must be non-nil +func (c *Commit) PatchContext(ctx context.Context, to *Commit) (*Patch, error) { fromTree, err := c.Tree() if err != nil { return nil, err @@ -86,7 +89,12 @@ func (c *Commit) Patch(to *Commit) (*Patch, error) { return nil, err } - return fromTree.Patch(toTree) + return fromTree.PatchContext(ctx, toTree) +} + +// Patch returns the Patch between the actual commit and the provided one. +func (c *Commit) Patch(to *Commit) (*Patch, error) { + return c.PatchContext(context.Background(), to) } // Parents return a CommitIter to the parent Commits. @@ -174,23 +182,13 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } if pgpsig { - // Check if it's the end of a PGP signature. - if bytes.Contains(line, []byte(endpgp)) { - c.PGPSignature += endpgp + "\n" - pgpsig = false - } else { - // Trim the left padding. + if len(line) > 0 && line[0] == ' ' { line = bytes.TrimLeft(line, " ") c.PGPSignature += string(line) + continue + } else { + pgpsig = false } - continue - } - - // Check if it's the beginning of a PGP signature. - if bytes.Contains(line, []byte(beginpgp)) { - c.PGPSignature += beginpgp + "\n" - pgpsig = true - continue } if !message { @@ -201,15 +199,24 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } split := bytes.SplitN(line, []byte{' '}, 2) + + var data []byte + if len(split) == 2 { + data = split[1] + } + switch string(split[0]) { case "tree": - c.TreeHash = plumbing.NewHash(string(split[1])) + c.TreeHash = plumbing.NewHash(string(data)) case "parent": - c.ParentHashes = append(c.ParentHashes, plumbing.NewHash(string(split[1]))) + c.ParentHashes = append(c.ParentHashes, plumbing.NewHash(string(data))) case "author": - c.Author.Decode(split[1]) + c.Author.Decode(data) case "committer": - c.Committer.Decode(split[1]) + c.Committer.Decode(data) + case headerpgp: + c.PGPSignature += string(data) + "\n" + pgpsig = true } } else { c.Message += string(line) @@ -262,17 +269,18 @@ func (b *Commit) encode(o plumbing.EncodedObject, includeSig bool) (err error) { } if b.PGPSignature != "" && includeSig { - if _, err = fmt.Fprint(w, "pgpsig"); err != nil { + if _, err = fmt.Fprint(w, "\n"+headerpgp+" "); err != nil { return err } - // Split all the signature lines and write with a left padding and - // newline at the end. - lines := strings.Split(b.PGPSignature, "\n") - for _, line := range lines { - if _, err = fmt.Fprintf(w, " %s\n", line); err != nil { - return err - } + // Split all the signature lines and re-write with a left padding and + // newline. Use join for this so it's clear that a newline should not be + // added after this section, as it will be added when the message is + // printed. + signature := strings.TrimSuffix(b.PGPSignature, "\n") + lines := strings.Split(signature, "\n") + if _, err = fmt.Fprint(w, strings.Join(lines, "\n ")); err != nil { + return err } } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_walker_file.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_walker_file.go new file mode 100644 index 00000000..84e738ac --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_walker_file.go @@ -0,0 +1,115 @@ +package object + +import ( + "gopkg.in/src-d/go-git.v4/plumbing/storer" + "io" +) + +type commitFileIter struct { + fileName string + sourceIter CommitIter + currentCommit *Commit +} + +// NewCommitFileIterFromIter returns a commit iterator which performs diffTree between +// successive trees returned from the commit iterator from the argument. The purpose of this is +// to find the commits that explain how the files that match the path came to be. +func NewCommitFileIterFromIter(fileName string, commitIter CommitIter) CommitIter { + iterator := new(commitFileIter) + iterator.sourceIter = commitIter + iterator.fileName = fileName + return iterator +} + +func (c *commitFileIter) Next() (*Commit, error) { + if c.currentCommit == nil { + var err error + c.currentCommit, err = c.sourceIter.Next() + if err != nil { + return nil, err + } + } + commit, commitErr := c.getNextFileCommit() + + // Setting current-commit to nil to prevent unwanted states when errors are raised + if commitErr != nil { + c.currentCommit = nil + } + return commit, commitErr +} + +func (c *commitFileIter) getNextFileCommit() (*Commit, error) { + for { + // Parent-commit can be nil if the current-commit is the initial commit + parentCommit, parentCommitErr := c.sourceIter.Next() + if parentCommitErr != nil { + // If the parent-commit is beyond the initial commit, keep it nil + if parentCommitErr != io.EOF { + return nil, parentCommitErr + } + parentCommit = nil + } + + // Fetch the trees of the current and parent commits + currentTree, currTreeErr := c.currentCommit.Tree() + if currTreeErr != nil { + return nil, currTreeErr + } + + var parentTree *Tree + if parentCommit != nil { + var parentTreeErr error + parentTree, parentTreeErr = parentCommit.Tree() + if parentTreeErr != nil { + return nil, parentTreeErr + } + } + + // Find diff between current and parent trees + changes, diffErr := DiffTree(currentTree, parentTree) + if diffErr != nil { + return nil, diffErr + } + + foundChangeForFile := false + for _, change := range changes { + if change.name() == c.fileName { + foundChangeForFile = true + break + } + } + + // Storing the current-commit in-case a change is found, and + // Updating the current-commit for the next-iteration + prevCommit := c.currentCommit + c.currentCommit = parentCommit + + if foundChangeForFile == true { + return prevCommit, nil + } + + // If not matches found and if parent-commit is beyond the initial commit, then return with EOF + if parentCommit == nil { + return nil, io.EOF + } + } +} + +func (c *commitFileIter) ForEach(cb func(*Commit) error) error { + for { + commit, nextErr := c.Next() + if nextErr != nil { + return nextErr + } + err := cb(commit) + if err == storer.ErrStop { + return nil + } else if err != nil { + return err + } + } +} + +func (c *commitFileIter) Close() { + c.sourceIter.Close() +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree.go index ac58c4dc..a30a29e3 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "gopkg.in/src-d/go-git.v4/utils/merkletrie" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" @@ -10,6 +11,13 @@ import ( // DiffTree compares the content and mode of the blobs found via two // tree objects. func DiffTree(a, b *Tree) (Changes, error) { + return DiffTreeContext(context.Background(), a, b) +} + +// DiffTree compares the content and mode of the blobs found via two +// tree objects. Provided context must be non-nil. +// An error will be return if context expires +func DiffTreeContext(ctx context.Context, a, b *Tree) (Changes, error) { from := NewTreeRootNode(a) to := NewTreeRootNode(b) @@ -17,8 +25,11 @@ func DiffTree(a, b *Tree) (Changes, error) { return bytes.Equal(a.Hash(), b.Hash()) } - merkletrieChanges, err := merkletrie.DiffTree(from, to, hashEqual) + merkletrieChanges, err := merkletrie.DiffTreeContext(ctx, from, to, hashEqual) if err != nil { + if err == merkletrie.ErrCanceled { + return nil, ErrCanceled + } return nil, err } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go index 4b59aba7..e960e50c 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go @@ -152,7 +152,11 @@ func (s *Signature) decodeTimeAndTimeZone(b []byte) { } func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error { - _, err := fmt.Fprintf(w, "%d %s", s.When.Unix(), s.When.Format("-0700")) + u := s.When.Unix() + if u < 0 { + u = 0 + } + _, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700")) return err } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch.go index aa96a96c..adeaccb0 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch.go @@ -2,6 +2,8 @@ package object import ( "bytes" + "context" + "errors" "fmt" "io" "math" @@ -15,10 +17,25 @@ import ( dmp "github.com/sergi/go-diff/diffmatchpatch" ) +var ( + ErrCanceled = errors.New("operation canceled") +) + func getPatch(message string, changes ...*Change) (*Patch, error) { + ctx := context.Background() + return getPatchContext(ctx, message, changes...) +} + +func getPatchContext(ctx context.Context, message string, changes ...*Change) (*Patch, error) { var filePatches []fdiff.FilePatch for _, c := range changes { - fp, err := filePatch(c) + select { + case <-ctx.Done(): + return nil, ErrCanceled + default: + } + + fp, err := filePatchWithContext(ctx, c) if err != nil { return nil, err } @@ -29,7 +46,7 @@ func getPatch(message string, changes ...*Change) (*Patch, error) { return &Patch{message, filePatches}, nil } -func filePatch(c *Change) (fdiff.FilePatch, error) { +func filePatchWithContext(ctx context.Context, c *Change) (fdiff.FilePatch, error) { from, to, err := c.Files() if err != nil { return nil, err @@ -52,6 +69,12 @@ func filePatch(c *Change) (fdiff.FilePatch, error) { var chunks []fdiff.Chunk for _, d := range diffs { + select { + case <-ctx.Done(): + return nil, ErrCanceled + default: + } + var op fdiff.Operation switch d.Type { case dmp.DiffEqual: @@ -70,6 +93,11 @@ func filePatch(c *Change) (fdiff.FilePatch, error) { from: c.From, to: c.To, }, nil + +} + +func filePatch(c *Change) (fdiff.FilePatch, error) { + return filePatchWithContext(context.Background(), c) } func fileContent(f *File) (content string, isBinary bool, err error) { diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag.go index 905206bd..03749f9a 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag.go @@ -195,13 +195,14 @@ func (t *Tag) encode(o plumbing.EncodedObject, includeSig bool) (err error) { return err } - if t.PGPSignature != "" && includeSig { - // Split all the signature lines and write with a newline at the end. - lines := strings.Split(t.PGPSignature, "\n") - for _, line := range lines { - if _, err = fmt.Fprintf(w, "%s\n", line); err != nil { - return err - } + // Note that this is highly sensitive to what it sent along in the message. + // Message *always* needs to end with a newline, or else the message and the + // signature will be concatenated into a corrupt object. Since this is a + // lower-level method, we assume you know what you are doing and have already + // done the needful on the message in the caller. + if includeSig { + if _, err = fmt.Fprint(w, t.PGPSignature); err != nil { + return err } } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree.go index 30bbcb03..78d61a1f 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree.go @@ -2,6 +2,7 @@ package object import ( "bufio" + "context" "errors" "fmt" "io" @@ -25,6 +26,7 @@ var ( ErrMaxTreeDepth = errors.New("maximum tree depth exceeded") ErrFileNotFound = errors.New("file not found") ErrDirectoryNotFound = errors.New("directory not found") + ErrEntryNotFound = errors.New("entry not found") ) // Tree is basically like a directory - it references a bunch of other trees @@ -85,6 +87,17 @@ func (t *Tree) File(path string) (*File, error) { return NewFile(path, e.Mode, blob), nil } +// Size returns the plaintext size of an object, without reading it +// into memory. +func (t *Tree) Size(path string) (int64, error) { + e, err := t.FindEntry(path) + if err != nil { + return 0, ErrEntryNotFound + } + + return t.s.EncodedObjectSize(e.Hash) +} + // Tree returns the tree identified by the `path` argument. // The path is interpreted as relative to the tree receiver. func (t *Tree) Tree(path string) (*Tree, error) { @@ -166,8 +179,6 @@ func (t *Tree) dir(baseName string) (*Tree, error) { return tree, err } -var errEntryNotFound = errors.New("entry not found") - func (t *Tree) entry(baseName string) (*TreeEntry, error) { if t.m == nil { t.buildMap() @@ -175,7 +186,7 @@ func (t *Tree) entry(baseName string) (*TreeEntry, error) { entry, ok := t.m[baseName] if !ok { - return nil, errEntryNotFound + return nil, ErrEntryNotFound } return entry, nil @@ -295,15 +306,30 @@ func (from *Tree) Diff(to *Tree) (Changes, error) { return DiffTree(from, to) } +// Diff returns a list of changes between this tree and the provided one +// Error will be returned if context expires +// Provided context must be non nil +func (from *Tree) DiffContext(ctx context.Context, to *Tree) (Changes, error) { + return DiffTreeContext(ctx, from, to) +} + // Patch returns a slice of Patch objects with all the changes between trees // in chunks. This representation can be used to create several diff outputs. func (from *Tree) Patch(to *Tree) (*Patch, error) { - changes, err := DiffTree(from, to) + return from.PatchContext(context.Background(), to) +} + +// Patch returns a slice of Patch objects with all the changes between trees +// in chunks. This representation can be used to create several diff outputs. +// If context expires, an error will be returned +// Provided context must be non-nil +func (from *Tree) PatchContext(ctx context.Context, to *Tree) (*Patch, error) { + changes, err := DiffTreeContext(ctx, from, to) if err != nil { return nil, err } - return changes.Patch() + return changes.PatchContext(ctx) } // treeEntryIter facilitates iterating through the TreeEntry objects in a Tree. diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/storer/object.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/storer/object.go index 92aa6291..2ac9b091 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/storer/object.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/storer/object.go @@ -40,6 +40,8 @@ type EncodedObjectStorer interface { // HasEncodedObject returns ErrObjNotFound if the object doesn't // exist. If the object does exist, it returns nil. HasEncodedObject(plumbing.Hash) error + // EncodedObjectSize returns the plaintext size of the encoded object. + EncodedObjectSize(plumbing.Hash) (int64, error) } // DeltaObjectStorer is an EncodedObjectStorer that can return delta diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/http/common.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/http/common.go index 2c337b77..c0348464 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/http/common.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/http/common.go @@ -201,6 +201,31 @@ func (a *BasicAuth) String() string { return fmt.Sprintf("%s - %s:%s", a.Name(), a.Username, masked) } +// TokenAuth implements the go-git http.AuthMethod and transport.AuthMethod interfaces +type TokenAuth struct { + Token string +} + +func (a *TokenAuth) setAuth(r *http.Request) { + if a == nil { + return + } + r.Header.Add("Authorization", fmt.Sprintf("Bearer %s", a.Token)) +} + +// Name is name of the auth +func (a *TokenAuth) Name() string { + return "http-token-auth" +} + +func (a *TokenAuth) String() string { + masked := "*******" + if a.Token == "" { + masked = "" + } + return fmt.Sprintf("%s - %s", a.Name(), masked) +} + // Err is a dedicated error to return errors based on status code type Err struct { Response *http.Response diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common/common.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common/common.go index 8ec1ea52..00497f3c 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common/common.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common/common.go @@ -382,6 +382,7 @@ var ( gitProtocolNotFoundErr = "ERR \n Repository not found." gitProtocolNoSuchErr = "ERR no such repository" gitProtocolAccessDeniedErr = "ERR access denied" + gogsAccessDeniedErr = "Gogs: Repository does not exist or you do not have access" ) func isRepoNotFoundError(s string) bool { @@ -409,6 +410,10 @@ func isRepoNotFoundError(s string) bool { return true } + if strings.HasPrefix(s, gogsAccessDeniedErr) { + return true + } + return false } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go index c83752c2..13b35262 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go @@ -1,6 +1,7 @@ package server import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -43,7 +44,7 @@ func (l *fsLoader) Load(ep *transport.Endpoint) (storer.Storer, error) { return nil, transport.ErrRepositoryNotFound } - return filesystem.NewStorage(fs) + return filesystem.NewStorage(fs, cache.NewObjectLRUDefault()), nil } // MapLoader is a Loader that uses a lookup map of storer.Storer by diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go index 84cfab2a..dbb47c56 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go @@ -236,7 +236,7 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // NewKnownHostsCallback returns ssh.HostKeyCallback based on a file based on a // known_hosts file. http://man.openbsd.org/sshd#SSH_KNOWN_HOSTS_FILE_FORMAT // -// If files is empty, the list of files will be read from the SSH_KNOWN_HOSTS +// If list of files is empty, then it will be read from the SSH_KNOWN_HOSTS // environment variable, example: // /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file // @@ -244,13 +244,15 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // ~/.ssh/known_hosts // /etc/ssh/ssh_known_hosts func NewKnownHostsCallback(files ...string) (ssh.HostKeyCallback, error) { - files, err := getDefaultKnownHostsFiles() - if err != nil { - return nil, err + var err error + + if len(files) == 0 { + if files, err = getDefaultKnownHostsFiles(); err != nil { + return nil, err + } } - files, err = filterKnownHostsFiles(files...) - if err != nil { + if files, err = filterKnownHostsFiles(files...); err != nil { return nil, err } diff --git a/vendor/gopkg.in/src-d/go-git.v4/prune.go b/vendor/gopkg.in/src-d/go-git.v4/prune.go index 04913d65..c840325f 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/prune.go +++ b/vendor/gopkg.in/src-d/go-git.v4/prune.go @@ -49,7 +49,7 @@ func (r *Repository) Prune(opt PruneOptions) error { } // Otherwise it is a candidate for pruning. // Check out for too new objects next. - if opt.OnlyObjectsOlderThan != (time.Time{}) { + if !opt.OnlyObjectsOlderThan.IsZero() { // Errors here are non-fatal. The object may be e.g. packed. // Or concurrently deleted. Skip such objects. t, err := los.LooseObjectTime(hash) diff --git a/vendor/gopkg.in/src-d/go-git.v4/references.go b/vendor/gopkg.in/src-d/go-git.v4/references.go index a1872a5a..5673ac13 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/references.go +++ b/vendor/gopkg.in/src-d/go-git.v4/references.go @@ -47,7 +47,9 @@ func (s commitSorterer) Len() int { } func (s commitSorterer) Less(i, j int) bool { - return s.l[i].Committer.When.Before(s.l[j].Committer.When) + return s.l[i].Committer.When.Before(s.l[j].Committer.When) || + s.l[i].Committer.When.Equal(s.l[j].Committer.When) && + s.l[i].Author.When.Before(s.l[j].Author.When) } func (s commitSorterer) Swap(i, j int) { diff --git a/vendor/gopkg.in/src-d/go-git.v4/remote.go b/vendor/gopkg.in/src-d/go-git.v4/remote.go index 60461d61..0556b989 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/remote.go +++ b/vendor/gopkg.in/src-d/go-git.v4/remote.go @@ -619,7 +619,7 @@ func getHaves( return result, nil } -const refspecTag = "+refs/tags/*:refs/tags/*" +const refspecAllTags = "+refs/tags/*:refs/tags/*" func calculateRefs( spec []config.RefSpec, @@ -627,17 +627,32 @@ func calculateRefs( tagMode TagMode, ) (memory.ReferenceStorage, error) { if tagMode == AllTags { - spec = append(spec, refspecTag) + spec = append(spec, refspecAllTags) } + refs := make(memory.ReferenceStorage) + for _, s := range spec { + if err := doCalculateRefs(s, remoteRefs, refs); err != nil { + return nil, err + } + } + + return refs, nil +} + +func doCalculateRefs( + s config.RefSpec, + remoteRefs storer.ReferenceStorer, + refs memory.ReferenceStorage, +) error { iter, err := remoteRefs.IterReferences() if err != nil { - return nil, err + return err } - refs := make(memory.ReferenceStorage) - return refs, iter.ForEach(func(ref *plumbing.Reference) error { - if !config.MatchAny(spec, ref.Name()) { + var matched bool + err = iter.ForEach(func(ref *plumbing.Reference) error { + if !s.Match(ref.Name()) { return nil } @@ -654,8 +669,23 @@ func calculateRefs( return nil } - return refs.SetReference(ref) + matched = true + if err := refs.SetReference(ref); err != nil { + return err + } + + if !s.IsWildcard() { + return storer.ErrStop + } + + return nil }) + + if !matched && !s.IsWildcard() { + return fmt.Errorf("couldn't find remote ref %q", s.Src()) + } + + return err } func getWants(localStorer storage.Storer, refs memory.ReferenceStorage) ([]plumbing.Hash, error) { diff --git a/vendor/gopkg.in/src-d/go-git.v4/repository.go b/vendor/gopkg.in/src-d/go-git.v4/repository.go index 717381bd..507ff441 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/repository.go +++ b/vendor/gopkg.in/src-d/go-git.v4/repository.go @@ -1,18 +1,22 @@ package git import ( + "bytes" "context" "errors" "fmt" stdioutil "io/ioutil" "os" + "path" "path/filepath" "strings" "time" + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/internal/revision" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -31,7 +35,12 @@ var ( // ErrBranchExists an error stating the specified branch already exists ErrBranchExists = errors.New("branch already exists") // ErrBranchNotFound an error stating the specified branch does not exist - ErrBranchNotFound = errors.New("branch not found") + ErrBranchNotFound = errors.New("branch not found") + // ErrTagExists an error stating the specified tag already exists + ErrTagExists = errors.New("tag already exists") + // ErrTagNotFound an error stating the specified tag does not exist + ErrTagNotFound = errors.New("tag not found") + ErrInvalidReference = errors.New("invalid reference, should be a tag or a branch") ErrRepositoryNotExists = errors.New("repository does not exist") ErrRepositoryAlreadyExists = errors.New("repository already exists") @@ -166,15 +175,6 @@ func Open(s storage.Storer, worktree billy.Filesystem) (*Repository, error) { return nil, err } - cfg, err := s.Config() - if err != nil { - return nil, err - } - - if !cfg.Core.IsBare && worktree == nil { - return nil, ErrWorktreeNotProvided - } - return newRepository(s, worktree), nil } @@ -220,10 +220,7 @@ func PlainInit(path string, isBare bool) (*Repository, error) { dot, _ = wt.Chroot(GitDirName) } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Init(s, wt) } @@ -235,9 +232,8 @@ func PlainOpen(path string) (*Repository, error) { return PlainOpenWithOptions(path, &PlainOpenOptions{}) } -// PlainOpen opens a git repository from the given path. It detects if the -// repository is bare or a normal one. If the path doesn't contain a valid -// repository ErrRepositoryNotExists is returned +// PlainOpenWithOptions opens a git repository from the given path with specific +// options. See PlainOpen for more info. func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) { dot, wt, err := dotGitToOSFilesystems(path, o.DetectDotGit) if err != nil { @@ -252,10 +248,7 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Open(s, wt) } @@ -333,6 +326,8 @@ func dotGitFileToOSFilesystem(path string, fs billy.Filesystem) (bfs billy.Files // PlainClone a repository into the path with the given options, isBare defines // if the new repository will be bare or normal. If the path is not empty // ErrRepositoryAlreadyExists is returned. +// +// TODO(mcuadros): move isBare to CloneOptions in v5 func PlainClone(path string, isBare bool, o *CloneOptions) (*Repository, error) { return PlainCloneContext(context.Background(), path, isBare, o) } @@ -344,6 +339,8 @@ func PlainClone(path string, isBare bool, o *CloneOptions) (*Repository, error) // The provided Context must be non-nil. If the context expires before the // operation is complete, an error is returned. The context only affects to the // transport operations. +// +// TODO(mcuadros): move isBare to CloneOptions in v5 func PlainCloneContext(ctx context.Context, path string, isBare bool, o *CloneOptions) (*Repository, error) { r, err := PlainInit(path, isBare) if err != nil { @@ -484,6 +481,139 @@ func (r *Repository) DeleteBranch(name string) error { return r.Storer.SetConfig(cfg) } +// CreateTag creates a tag. If opts is included, the tag is an annotated tag, +// otherwise a lightweight tag is created. +func (r *Repository) CreateTag(name string, hash plumbing.Hash, opts *CreateTagOptions) (*plumbing.Reference, error) { + rname := plumbing.ReferenceName(path.Join("refs", "tags", name)) + + _, err := r.Storer.Reference(rname) + switch err { + case nil: + // Tag exists, this is an error + return nil, ErrTagExists + case plumbing.ErrReferenceNotFound: + // Tag missing, available for creation, pass this + default: + // Some other error + return nil, err + } + + var target plumbing.Hash + if opts != nil { + target, err = r.createTagObject(name, hash, opts) + if err != nil { + return nil, err + } + } else { + target = hash + } + + ref := plumbing.NewHashReference(rname, target) + if err = r.Storer.SetReference(ref); err != nil { + return nil, err + } + + return ref, nil +} + +func (r *Repository) createTagObject(name string, hash plumbing.Hash, opts *CreateTagOptions) (plumbing.Hash, error) { + if err := opts.Validate(r, hash); err != nil { + return plumbing.ZeroHash, err + } + + rawobj, err := object.GetObject(r.Storer, hash) + if err != nil { + return plumbing.ZeroHash, err + } + + tag := &object.Tag{ + Name: name, + Tagger: *opts.Tagger, + Message: opts.Message, + TargetType: rawobj.Type(), + Target: hash, + } + + if opts.SignKey != nil { + sig, err := r.buildTagSignature(tag, opts.SignKey) + if err != nil { + return plumbing.ZeroHash, err + } + + tag.PGPSignature = sig + } + + obj := r.Storer.NewEncodedObject() + if err := tag.Encode(obj); err != nil { + return plumbing.ZeroHash, err + } + + return r.Storer.SetEncodedObject(obj) +} + +func (r *Repository) buildTagSignature(tag *object.Tag, signKey *openpgp.Entity) (string, error) { + encoded := &plumbing.MemoryObject{} + if err := tag.Encode(encoded); err != nil { + return "", err + } + + rdr, err := encoded.Reader() + if err != nil { + return "", err + } + + var b bytes.Buffer + if err := openpgp.ArmoredDetachSign(&b, signKey, rdr, nil); err != nil { + return "", err + } + + return b.String(), nil +} + +// Tag returns a tag from the repository. +// +// If you want to check to see if the tag is an annotated tag, you can call +// TagObject on the hash of the reference in ForEach: +// +// ref, err := r.Tag("v0.1.0") +// if err != nil { +// // Handle error +// } +// +// obj, err := r.TagObject(ref.Hash()) +// switch err { +// case nil: +// // Tag object present +// case plumbing.ErrObjectNotFound: +// // Not a tag object +// default: +// // Some other error +// } +// +func (r *Repository) Tag(name string) (*plumbing.Reference, error) { + ref, err := r.Reference(plumbing.ReferenceName(path.Join("refs", "tags", name)), false) + if err != nil { + if err == plumbing.ErrReferenceNotFound { + // Return a friendly error for this one, versus just ReferenceNotFound. + return nil, ErrTagNotFound + } + + return nil, err + } + + return ref, nil +} + +// DeleteTag deletes a tag from the repository. +func (r *Repository) DeleteTag(name string) error { + _, err := r.Tag(name) + if err != nil { + return err + } + + return r.Storer.RemoveReference(plumbing.ReferenceName(path.Join("refs", "tags", name))) +} + func (r *Repository) resolveToCommitHash(h plumbing.Hash) (plumbing.Hash, error) { obj, err := r.Storer.EncodedObject(plumbing.AnyObject, h) if err != nil { @@ -519,11 +649,12 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } ref, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ - RefSpecs: r.cloneRefSpec(o, c), - Depth: o.Depth, - Auth: o.Auth, - Progress: o.Progress, - Tags: o.Tags, + RefSpecs: r.cloneRefSpec(o, c), + Depth: o.Depth, + Auth: o.Auth, + Progress: o.Progress, + Tags: o.Tags, + RemoteName: o.RemoteName, }, o.ReferenceName) if err != nil { return err @@ -583,7 +714,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } const ( - refspecTagWithDepth = "+refs/tags/%s:refs/tags/%[1]s" + refspecTag = "+refs/tags/%s:refs/tags/%[1]s" refspecSingleBranch = "+refs/heads/%s:refs/remotes/%s/%[1]s" refspecSingleBranchHEAD = "+HEAD:refs/remotes/%s/HEAD" ) @@ -592,8 +723,8 @@ func (r *Repository) cloneRefSpec(o *CloneOptions, c *config.RemoteConfig) []con var rs string switch { - case o.ReferenceName.IsTag() && o.Depth > 0: - rs = fmt.Sprintf(refspecTagWithDepth, o.ReferenceName.Short()) + case o.ReferenceName.IsTag(): + rs = fmt.Sprintf(refspecTag, o.ReferenceName.Short()) case o.SingleBranch && o.ReferenceName == plumbing.HEAD: rs = fmt.Sprintf(refspecSingleBranchHEAD, c.Name) case o.SingleBranch: @@ -830,23 +961,53 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { return nil, err } + var commitIter object.CommitIter switch o.Order { case LogOrderDefault: - return object.NewCommitPreorderIter(commit, nil, nil), nil + commitIter = object.NewCommitPreorderIter(commit, nil, nil) case LogOrderDFS: - return object.NewCommitPreorderIter(commit, nil, nil), nil + commitIter = object.NewCommitPreorderIter(commit, nil, nil) case LogOrderDFSPost: - return object.NewCommitPostorderIter(commit, nil), nil + commitIter = object.NewCommitPostorderIter(commit, nil) case LogOrderBSF: - return object.NewCommitIterBSF(commit, nil, nil), nil + commitIter = object.NewCommitIterBSF(commit, nil, nil) case LogOrderCommitterTime: - return object.NewCommitIterCTime(commit, nil, nil), nil + commitIter = object.NewCommitIterCTime(commit, nil, nil) + default: + return nil, fmt.Errorf("invalid Order=%v", o.Order) + } + + if o.FileName == nil { + return commitIter, nil } - return nil, fmt.Errorf("invalid Order=%v", o.Order) + return object.NewCommitFileIterFromIter(*o.FileName, commitIter), nil } -// Tags returns all the References from Tags. This method returns all the tag -// types, lightweight, and annotated ones. +// Tags returns all the tag References in a repository. +// +// If you want to check to see if the tag is an annotated tag, you can call +// TagObject on the hash Reference passed in through ForEach: +// +// iter, err := r.Tags() +// if err != nil { +// // Handle error +// } +// +// if err := iter.ForEach(func (ref *plumbing.Reference) error { +// obj, err := r.TagObject(ref.Hash()) +// switch err { +// case nil: +// // Tag object present +// case plumbing.ErrObjectNotFound: +// // Not a tag object +// default: +// // Some other error +// return err +// } +// }); err != nil { +// // Handle outer iterator error +// } +// func (r *Repository) Tags() (storer.ReferenceIter, error) { refIter, err := r.Storer.IterReferences() if err != nil { @@ -872,7 +1033,8 @@ func (r *Repository) Branches() (storer.ReferenceIter, error) { }, refIter), nil } -// Notes returns all the References that are Branches. +// Notes returns all the References that are notes. For more information: +// https://git-scm.com/docs/git-notes func (r *Repository) Notes() (storer.ReferenceIter, error) { refIter, err := r.Storer.IterReferences() if err != nil { @@ -1004,7 +1166,18 @@ func (r *Repository) Worktree() (*Worktree, error) { return &Worktree{r: r, Filesystem: r.wt}, nil } -// ResolveRevision resolves revision to corresponding hash. +func countTrue(vals ...bool) int { + sum := 0 + for _, v := range vals { + if v { + sum++ + } + } + return sum +} + +// ResolveRevision resolves revision to corresponding hash. It will always +// resolve to a commit hash, not a tree or annotated tag. // // Implemented resolvers : HEAD, branch, tag, heads/branch, refs/heads/branch, // refs/tags/tag, refs/remotes/origin/branch, refs/remotes/origin/HEAD, tilde and caret (HEAD~1, master~^, tag~2, ref/heads/master~1, ...), selection by text (HEAD^{/fix nasty bug}) @@ -1024,8 +1197,8 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err case revision.Ref: revisionRef := item.(revision.Ref) var ref *plumbing.Reference - var hashCommit, refCommit *object.Commit - var rErr, hErr error + var hashCommit, refCommit, tagCommit *object.Commit + var rErr, hErr, tErr error for _, rule := range append([]string{"%s"}, plumbing.RefRevParseRules...) { ref, err = storer.ResolveReference(r.Storer, plumbing.ReferenceName(fmt.Sprintf(rule, revisionRef))) @@ -1036,24 +1209,38 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err } if ref != nil { + tag, tObjErr := r.TagObject(ref.Hash()) + if tObjErr != nil { + tErr = tObjErr + } else { + tagCommit, tErr = tag.Commit() + } refCommit, rErr = r.CommitObject(ref.Hash()) } else { rErr = plumbing.ErrReferenceNotFound + tErr = plumbing.ErrReferenceNotFound } - isHash := plumbing.NewHash(string(revisionRef)).String() == string(revisionRef) - - if isHash { + maybeHash := plumbing.NewHash(string(revisionRef)).String() == string(revisionRef) + if maybeHash { hashCommit, hErr = r.CommitObject(plumbing.NewHash(string(revisionRef))) + } else { + hErr = plumbing.ErrReferenceNotFound } + isTag := tErr == nil + isCommit := rErr == nil + isHash := hErr == nil + switch { - case rErr == nil && !isHash: + case countTrue(isTag, isCommit, isHash) > 1: + return &plumbing.ZeroHash, fmt.Errorf(`refname "%s" is ambiguous`, revisionRef) + case isTag: + commit = tagCommit + case isCommit: commit = refCommit - case rErr != nil && isHash && hErr == nil: + case isHash: commit = hashCommit - case rErr == nil && isHash && hErr == nil: - return &plumbing.ZeroHash, fmt.Errorf(`refname "%s" is ambiguous`, revisionRef) default: return &plumbing.ZeroHash, plumbing.ErrReferenceNotFound } diff --git a/vendor/gopkg.in/src-d/go-git.v4/status.go b/vendor/gopkg.in/src-d/go-git.v4/status.go index ef8a500f..7f18e022 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/status.go +++ b/vendor/gopkg.in/src-d/go-git.v4/status.go @@ -1,7 +1,10 @@ package git -import "fmt" -import "bytes" +import ( + "bytes" + "fmt" + "path/filepath" +) // Status represents the current status of a Worktree. // The key of the map is the path of the file. @@ -17,7 +20,13 @@ func (s Status) File(path string) *FileStatus { return s[path] } -// IsClean returns true if all the files aren't in Unmodified status. +// IsUntracked checks if file for given path is 'Untracked' +func (s Status) IsUntracked(path string) bool { + stat, ok := (s)[filepath.ToSlash(path)] + return ok && stat.Worktree == Untracked +} + +// IsClean returns true if all the files are in Unmodified status. func (s Status) IsClean() bool { for _, status := range s { if status.Worktree != Unmodified || status.Staging != Unmodified { diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go index 52b621c5..a58c2482 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go @@ -57,17 +57,48 @@ var ( ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") ) +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem + options Options + fs billy.Filesystem + + // incoming object directory information + incomingChecked bool + incomingDirName string + + objectList []plumbing.Hash + objectMap map[plumbing.Hash]struct{} + packList []plumbing.Hash + packMap map[plumbing.Hash]struct{} + + files map[string]billy.File } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return NewWithOptions(fs, Options{}) +} + +// NewWithOptions sets non default configuration options. +// See New for complete help. +func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { + return &DotGit{ + options: o, + fs: fs, + } } // Initialize creates all the folder scaffolding. @@ -97,6 +128,28 @@ func (d *DotGit) Initialize() error { return nil } +// Close closes all opened files. +func (d *DotGit) Close() error { + var firstError error + if d.files != nil { + for _, f := range d.files { + err := f.Close() + if err != nil && firstError == nil { + firstError = err + continue + } + } + + d.files = nil + } + + if firstError != nil { + return firstError + } + + return nil +} + // ConfigWriter returns a file pointer for write to the config file func (d *DotGit) ConfigWriter() (billy.File, error) { return d.fs.Create(configPath) @@ -139,11 +192,25 @@ func (d *DotGit) Shallow() (billy.File, error) { // NewObjectPack return a writer for a new packfile, it saves the packfile to // disk and also generates and save the index for the given packfile. func (d *DotGit) NewObjectPack() (*PackWriter, error) { + d.cleanPackList() return newPackWrite(d.fs) } // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { + if !d.options.ExclusiveAccess { + return d.objectPacks() + } + + err := d.genPackList() + if err != nil { + return nil, err + } + + return d.packList, nil +} + +func (d *DotGit) objectPacks() ([]plumbing.Hash, error) { packDir := d.fs.Join(objectsPath, packPath) files, err := d.fs.ReadDir(packDir) if err != nil { @@ -177,7 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { - pack, err := d.fs.Open(d.objectPackPath(hash, extension)) + if d.files == nil { + d.files = make(map[string]billy.File) + } + + err := d.hasPack(hash) + if err != nil { + return nil, err + } + + path := d.objectPackPath(hash, extension) + f, ok := d.files[path] + if ok { + return f, nil + } + + pack, err := d.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrPackfileNotFound @@ -186,20 +268,36 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil return nil, err } + if d.options.KeepDescriptors && extension == "pack" { + d.files[path] = pack + } + return pack, nil } // ObjectPack returns a fs.File of the given packfile func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `pack`) } // ObjectPackIdx returns a fs.File of the index file for a given packfile func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `idx`) } func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error { + d.cleanPackList() + path := d.objectPackPath(hash, `pack`) if !t.IsZero() { fi, err := d.fs.Stat(path) @@ -220,12 +318,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er // NewObject return a writer for a new object file. func (d *DotGit) NewObject() (*ObjectWriter, error) { + d.cleanObjectList() + return newObjectWriter(d.fs) } // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { + if d.options.ExclusiveAccess { + err := d.genObjectList() + if err != nil { + return nil, err + } + + return d.objectList, nil + } + var objects []plumbing.Hash err := d.ForEachObjectHash(func(hash plumbing.Hash) error { objects = append(objects, hash) @@ -237,9 +346,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { return objects, nil } -// Objects returns a slice with the hashes of objects found under the -// .git/objects/ directory. +// ForEachObjectHash iterates over the hashes of objects found under the +// .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { + if !d.options.ExclusiveAccess { + return d.forEachObjectHash(fun) + } + + err := d.genObjectList() + if err != nil { + return err + } + + for _, h := range d.objectList { + err := fun(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error { files, err := d.fs.ReadDir(objectsPath) if err != nil { if os.IsNotExist(err) { @@ -274,24 +403,178 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { return nil } +func (d *DotGit) cleanObjectList() { + d.objectMap = nil + d.objectList = nil +} + +func (d *DotGit) genObjectList() error { + if d.objectMap != nil { + return nil + } + + d.objectMap = make(map[plumbing.Hash]struct{}) + return d.forEachObjectHash(func(h plumbing.Hash) error { + d.objectList = append(d.objectList, h) + d.objectMap[h] = struct{}{} + + return nil + }) +} + +func (d *DotGit) hasObject(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genObjectList() + if err != nil { + return err + } + + _, ok := d.objectMap[h] + if !ok { + return plumbing.ErrObjectNotFound + } + + return nil +} + +func (d *DotGit) cleanPackList() { + d.packMap = nil + d.packList = nil +} + +func (d *DotGit) genPackList() error { + if d.packMap != nil { + return nil + } + + op, err := d.objectPacks() + if err != nil { + return err + } + + d.packMap = make(map[plumbing.Hash]struct{}) + d.packList = nil + + for _, h := range op { + d.packList = append(d.packList, h) + d.packMap[h] = struct{}{} + } + + return nil +} + +func (d *DotGit) hasPack(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genPackList() + if err != nil { + return err + } + + _, ok := d.packMap[h] + if !ok { + return ErrPackfileNotFound + } + + return nil +} + func (d *DotGit) objectPath(h plumbing.Hash) string { hash := h.String() return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } +// incomingObjectPath is intended to add support for a git pre-receive hook +// to be written it adds support for go-git to find objects in an "incoming" +// directory, so that the library can be used to write a pre-receive hook +// that deals with the incoming objects. +// +// More on git hooks found here : https://git-scm.com/docs/githooks +// More on 'quarantine'/incoming directory here: +// https://git-scm.com/docs/git-receive-pack +func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { + hString := h.String() + + if d.incomingDirName == "" { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) +} + +// hasIncomingObjects searches for an incoming directory and keeps its name +// so it doesn't have to be found each time an object is accessed. +func (d *DotGit) hasIncomingObjects() bool { + if !d.incomingChecked { + directoryContents, err := d.fs.ReadDir(objectsPath) + if err == nil { + for _, file := range directoryContents { + if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() { + d.incomingDirName = file.Name() + } + } + } + + d.incomingChecked = true + } + + return d.incomingDirName != "" +} + // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { - return d.fs.Open(d.objectPath(h)) + err := d.hasObject(h) + if err != nil { + return nil, err + } + + obj1, err1 := d.fs.Open(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { - return d.fs.Stat(d.objectPath(h)) + err := d.hasObject(h) + if err != nil { + return nil, err + } + + obj1, err1 := d.fs.Stat(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { - return d.fs.Remove(d.objectPath(h)) + d.cleanObjectList() + + err1 := d.fs.Remove(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + err2 := d.fs.Remove(d.incomingObjectPath(h)) + if err2 != nil { + return err1 + } + return err2 + } + return err1 } func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { @@ -469,7 +752,7 @@ func (d *DotGit) openAndLockPackedRefs(doCreate bool) ( // File mode is retrieved from a constant defined in the target specific // files (dotgit_rewrite_packed_refs_*). Some modes are not available // in all filesystems. - openFlags := openAndLockPackedRefsMode + openFlags := d.openAndLockPackedRefsMode() if doCreate { openFlags |= os.O_CREATE } @@ -784,6 +1067,11 @@ func (d *DotGit) Alternates() ([]*DotGit, error) { return alternates, nil } +// Fs returns the underlying filesystem of the DotGit folder. +func (d *DotGit) Fs() billy.Filesystem { + return d.fs +} + func isHex(s string) bool { for _, b := range []byte(s) { if isNum(b) { diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go new file mode 100644 index 00000000..7f1c02c1 --- /dev/null +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go @@ -0,0 +1,81 @@ +package dotgit + +import ( + "io" + "os" + "runtime" + + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +func (d *DotGit) openAndLockPackedRefsMode() int { + if billy.CapabilityCheck(d.fs, billy.ReadAndWriteCapability) { + return os.O_RDWR + } + + return os.O_RDONLY +} + +func (d *DotGit) rewritePackedRefsWhileLocked( + tmp billy.File, pr billy.File) error { + // Try plain rename. If we aren't using the bare Windows filesystem as the + // storage layer, we might be able to get away with a rename over a locked + // file. + err := d.fs.Rename(tmp.Name(), pr.Name()) + if err == nil { + return nil + } + + // If we are in a filesystem that does not support rename (e.g. sivafs) + // a full copy is done. + if err == billy.ErrNotSupported { + return d.copyNewFile(tmp, pr) + } + + if runtime.GOOS != "windows" { + return err + } + + // Otherwise, Windows doesn't let us rename over a locked file, so + // we have to do a straight copy. Unfortunately this could result + // in a partially-written file if the process fails before the + // copy completes. + return d.copyToExistingFile(tmp, pr) +} + +func (d *DotGit) copyToExistingFile(tmp, pr billy.File) error { + _, err := pr.Seek(0, io.SeekStart) + if err != nil { + return err + } + err = pr.Truncate(0) + if err != nil { + return err + } + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = io.Copy(pr, tmp) + + return err +} + +func (d *DotGit) copyNewFile(tmp billy.File, pr billy.File) (err error) { + prWrite, err := d.fs.Create(pr.Name()) + if err != nil { + return err + } + + defer ioutil.CheckClose(prWrite, &err) + + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return err + } + + _, err = io.Copy(prWrite, tmp) + + return err +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go deleted file mode 100644 index c7607932..00000000 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build !windows,!norwfs - -package dotgit - -import ( - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDWR - -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - // On non-Windows platforms, we can have atomic rename. - return d.fs.Rename(tmp.Name(), pr.Name()) -} diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go deleted file mode 100644 index 6e43b428..00000000 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build norwfs - -package dotgit - -import ( - "io" - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDONLY - -// Instead of renaming that can not be supported in simpler filesystems -// a full copy is done. -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - - prWrite, err := d.fs.Create(pr.Name()) - if err != nil { - return err - } - - defer prWrite.Close() - - _, err = tmp.Seek(0, io.SeekStart) - if err != nil { - return err - } - - _, err = io.Copy(prWrite, tmp) - - return err -} diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go deleted file mode 100644 index 897d2c94..00000000 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go +++ /dev/null @@ -1,42 +0,0 @@ -// +build windows,!norwfs - -package dotgit - -import ( - "io" - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDWR - -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - // If we aren't using the bare Windows filesystem as the storage - // layer, we might be able to get away with a rename over a locked - // file. - err := d.fs.Rename(tmp.Name(), pr.Name()) - if err == nil { - return nil - } - - // Otherwise, Windows doesn't let us rename over a locked file, so - // we have to do a straight copy. Unfortunately this could result - // in a partially-written file if the process fails before the - // copy completes. - _, err = pr.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = pr.Truncate(0) - if err != nil { - return err - } - _, err = tmp.Seek(0, io.SeekStart) - if err != nil { - return err - } - _, err = io.Copy(pr, tmp) - return err -} diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/writers.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/writers.go index c2b420f8..93d2d8cc 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/writers.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/writers.go @@ -20,13 +20,14 @@ import ( // is renamed/moved (depends on the Filesystem implementation) to the final // location, if the PackWriter is not used, nothing is written type PackWriter struct { - Notify func(plumbing.Hash, *packfile.Index) + Notify func(plumbing.Hash, *idxfile.Writer) fs billy.Filesystem fr, fw billy.File synced *syncedReader checksum plumbing.Hash - index *packfile.Index + parser *packfile.Parser + writer *idxfile.Writer result chan error } @@ -55,20 +56,21 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) { func (w *PackWriter) buildIndex() { s := packfile.NewScanner(w.synced) - d, err := packfile.NewDecoder(s, nil) + w.writer = new(idxfile.Writer) + var err error + w.parser, err = packfile.NewParser(s, w.writer) if err != nil { w.result <- err return } - checksum, err := d.Decode() + checksum, err := w.parser.Parse() if err != nil { w.result <- err return } w.checksum = checksum - w.index = d.Index() w.result <- err } @@ -92,8 +94,8 @@ func (w *PackWriter) Write(p []byte) (int, error) { // was written, the tempfiles are deleted without writing a packfile. func (w *PackWriter) Close() error { defer func() { - if w.Notify != nil && w.index != nil && w.index.Size() > 0 { - w.Notify(w.checksum, w.index) + if w.Notify != nil && w.writer != nil && w.writer.Finished() { + w.Notify(w.checksum, w.writer) } close(w.result) @@ -115,7 +117,7 @@ func (w *PackWriter) Close() error { return err } - if w.index == nil || w.index.Size() == 0 { + if w.writer == nil || !w.writer.Finished() { return w.clean() } @@ -145,11 +147,13 @@ func (w *PackWriter) save() error { } func (w *PackWriter) encodeIdx(writer io.Writer) error { - idx := w.index.ToIdxFile() - idx.PackfileChecksum = w.checksum - idx.Version = idxfile.VersionSupported + idx, err := w.writer.Index() + if err != nil { + return err + } + e := idxfile.NewEncoder(writer) - _, err := e.Encode(idx) + _, err = e.Encode(idx) return err } @@ -209,7 +213,6 @@ func (s *syncedReader) isBlocked() bool { func (s *syncedReader) wake() { if s.isBlocked() { - // fmt.Println("wake") atomic.StoreUint32(&s.blocked, 0) s.news <- true } @@ -220,7 +223,6 @@ func (s *syncedReader) sleep() { written := atomic.LoadUint64(&s.written) if read >= written { atomic.StoreUint32(&s.blocked, 1) - // fmt.Println("sleep", read, written) <-s.news } diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go index 7c8c8d86..92722067 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go @@ -1,6 +1,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) @@ -15,5 +16,5 @@ func (s *ModuleStorage) Module(name string) (storage.Storer, error) { return nil, err } - return NewStorage(fs) + return NewStorage(fs, cache.NewObjectLRUDefault()), nil } diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go index 54f268aa..6cd2d4c8 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go @@ -12,27 +12,33 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" - "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" ) type ObjectStorage struct { + options Options + // deltaBaseCache is an object cache uses to cache delta's bases when deltaBaseCache cache.Object dir *dotgit.DotGit - index map[plumbing.Hash]*packfile.Index + index map[plumbing.Hash]idxfile.Index +} + +// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. +func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, cache, Options{}) } -func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { - s := ObjectStorage{ - deltaBaseCache: cache.NewObjectLRUDefault(), +// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options +func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { + return &ObjectStorage{ + options: ops, + deltaBaseCache: cache, dir: dir, } - - return s, nil } func (s *ObjectStorage) requireIndex() error { @@ -40,7 +46,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]*packfile.Index) + s.index = make(map[plumbing.Hash]idxfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -62,13 +68,14 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { } defer ioutil.CheckClose(f, &err) - idxf := idxfile.NewIdxfile() + + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { return err } - s.index[h] = packfile.NewIndexFromIdxFile(idxf) + s.index[h] = idxf return err } @@ -86,8 +93,11 @@ func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { return nil, err } - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { - s.index[h] = idx + w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) { + index, err := writer.Index() + if err == nil { + s.index[h] = index + } } return w, nil @@ -150,6 +160,79 @@ func (s *ObjectStorage) HasEncodedObject(h plumbing.Hash) (err error) { return nil } +func (s *ObjectStorage) encodedObjectSizeFromUnpacked(h plumbing.Hash) ( + size int64, err error) { + f, err := s.dir.Object(h) + if err != nil { + if os.IsNotExist(err) { + return 0, plumbing.ErrObjectNotFound + } + + return 0, err + } + + r, err := objfile.NewReader(f) + if err != nil { + return 0, err + } + defer ioutil.CheckClose(r, &err) + + _, size, err = r.Header() + return size, err +} + +func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( + size int64, err error) { + if err := s.requireIndex(); err != nil { + return 0, err + } + + pack, _, offset := s.findObjectInPackfile(h) + if offset == -1 { + return 0, plumbing.ErrObjectNotFound + } + + f, err := s.dir.ObjectPack(pack) + if err != nil { + return 0, err + } + defer ioutil.CheckClose(f, &err) + + idx := s.index[pack] + hash, err := idx.FindHash(offset) + if err == nil { + obj, ok := s.deltaBaseCache.Get(hash) + if ok { + return obj.Size(), nil + } + } else if err != nil && err != plumbing.ErrObjectNotFound { + return 0, err + } + + var p *packfile.Packfile + if s.deltaBaseCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + } else { + p = packfile.NewPackfile(idx, s.dir.Fs(), f) + } + + return p.GetSizeByOffset(offset) +} + +// EncodedObjectSize returns the plaintext size of the given object, +// without actually reading the full object data from storage. +func (s *ObjectStorage) EncodedObjectSize(h plumbing.Hash) ( + size int64, err error) { + size, err = s.encodedObjectSizeFromUnpacked(h) + if err != nil && err != plumbing.ErrObjectNotFound { + return 0, err + } else if err == nil { + return size, nil + } + + return s.encodedObjectSizeFromPackfile(h) +} + // EncodedObject returns the object with the given hash, by searching for it in // the packfile and the git object directories. func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { @@ -166,10 +249,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o, oe := newObjectStorage(dg) - if oe != nil { - continue - } + o := NewObjectStorage(dg, s.deltaBaseCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue @@ -265,7 +345,9 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( return nil, err } - defer ioutil.CheckClose(f, &err) + if !s.options.KeepDescriptors { + defer ioutil.CheckClose(f, &err) + } idx := s.index[pack] if canBeDelta { @@ -277,30 +359,37 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( func (s *ObjectStorage) decodeObjectAt( f billy.File, - idx *packfile.Index, - offset int64) (plumbing.EncodedObject, error) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, err + idx idxfile.Index, + offset int64, +) (plumbing.EncodedObject, error) { + hash, err := idx.FindHash(offset) + if err == nil { + obj, ok := s.deltaBaseCache.Get(hash) + if ok { + return obj, nil + } } - p := packfile.NewScanner(f) - - d, err := packfile.NewDecoderWithCache(p, memory.NewStorage(), - s.deltaBaseCache) - if err != nil { + if err != nil && err != plumbing.ErrObjectNotFound { return nil, err } - d.SetIndex(idx) - obj, err := d.DecodeObjectAt(offset) - return obj, err + var p *packfile.Packfile + if s.deltaBaseCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + } else { + p = packfile.NewPackfile(idx, s.dir.Fs(), f) + } + + return p.GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, - idx *packfile.Index, + idx idxfile.Index, offset int64, - hash plumbing.Hash) (plumbing.EncodedObject, error) { + hash plumbing.Hash, +) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err } @@ -323,12 +412,10 @@ func (s *ObjectStorage) decodeDeltaObjectAt( case plumbing.REFDeltaObject: base = header.Reference case plumbing.OFSDeltaObject: - e, ok := idx.LookupOffset(uint64(header.OffsetReference)) - if !ok { - return nil, plumbing.ErrObjectNotFound + base, err = idx.FindHash(header.OffsetReference) + if err != nil { + return nil, err } - - base = e.Hash default: return s.decodeObjectAt(f, idx, offset) } @@ -349,8 +436,9 @@ func (s *ObjectStorage) decodeDeltaObjectAt( func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) { for packfile, index := range s.index { - if e, ok := index.LookupHash(h); ok { - return packfile, e.Hash, int64(e.Offset) + offset, err := index.FindOffset(h) + if err == nil { + return packfile, h, offset } } @@ -381,7 +469,10 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode return storer.NewMultiEncodedObjectIter(iters), nil } -func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumbing.Hash]struct{}) (storer.EncodedObjectIter, error) { +func (s *ObjectStorage) buildPackfileIters( + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, +) (storer.EncodedObjectIter, error) { if err := s.requireIndex(); err != nil { return nil, err } @@ -397,11 +488,19 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb if err != nil { return nil, err } - return newPackfileIter(pack, t, seen, s.index[h], s.deltaBaseCache) + return newPackfileIter( + s.dir.Fs(), pack, t, seen, s.index[h], + s.deltaBaseCache, s.options.KeepDescriptors, + ) }, }, nil } +// Close closes all opened files. +func (s *ObjectStorage) Close() error { + return s.dir.Close() +} + type lazyPackfilesIter struct { hashes []plumbing.Hash open func(h plumbing.Hash) (storer.EncodedObjectIter, error) @@ -450,76 +549,104 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { - f billy.File - d *packfile.Decoder - t plumbing.ObjectType + pack billy.File + iter storer.EncodedObjectIter + seen map[plumbing.Hash]struct{} - seen map[plumbing.Hash]struct{} - position uint32 - total uint32 + // tells whether the pack file should be left open after iteration or not + keepPack bool } -func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) { - return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), nil, nil) -} +// NewPackfileIter returns a new EncodedObjectIter for the provided packfile +// and object type. Packfile and index file will be closed after they're +// used. If keepPack is true the packfile won't be closed after the iteration +// finished. +func NewPackfileIter( + fs billy.Filesystem, + f billy.File, + idxFile billy.File, + t plumbing.ObjectType, + keepPack bool, +) (storer.EncodedObjectIter, error) { + idx := idxfile.NewMemoryIndex() + if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { + return nil, err + } -func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, - index *packfile.Index, cache cache.Object) (storer.EncodedObjectIter, error) { - s := packfile.NewScanner(f) - _, total, err := s.Header() - if err != nil { + if err := idxFile.Close(); err != nil { return nil, err } - d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t, cache) + seen := make(map[plumbing.Hash]struct{}) + return newPackfileIter(fs, f, t, seen, idx, nil, keepPack) +} + +func newPackfileIter( + fs billy.Filesystem, + f billy.File, + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, + index idxfile.Index, + cache cache.Object, + keepPack bool, +) (storer.EncodedObjectIter, error) { + var p *packfile.Packfile + if cache != nil { + p = packfile.NewPackfileWithCache(index, fs, f, cache) + } else { + p = packfile.NewPackfile(index, fs, f) + } + + iter, err := p.GetByType(t) if err != nil { return nil, err } - d.SetIndex(index) - return &packfileIter{ - f: f, - d: d, - t: t, - - total: total, - seen: seen, + pack: f, + iter: iter, + seen: seen, + keepPack: keepPack, }, nil } func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { for { - if iter.position >= iter.total { - return nil, io.EOF - } - - obj, err := iter.d.DecodeObject() + obj, err := iter.iter.Next() if err != nil { return nil, err } - iter.position++ - if obj == nil { - continue - } - if _, ok := iter.seen[obj.Hash()]; ok { - return iter.Next() + continue } return obj, nil } } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + iter.Close() + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *packfileIter) Close() { - iter.f.Close() - iter.d.Close() + iter.iter.Close() + if !iter.keepPack { + _ = iter.pack.Close() + } } type objectsIter struct { @@ -547,9 +674,20 @@ func (iter *objectsIter) Next() (plumbing.EncodedObject, error) { return obj, err } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *objectsIter) Close() { diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go index d7aa18bc..14a772ab 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -22,25 +23,45 @@ type Storage struct { ModuleStorage } -// NewStorage returns a new Storage backed by a given `fs.Filesystem` -func NewStorage(fs billy.Filesystem) (*Storage, error) { - dir := dotgit.New(fs) - o, err := newObjectStorage(dir) - if err != nil { - return nil, err +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + +// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. +func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { + return NewStorageWithOptions(fs, cache, Options{}) +} + +// NewStorageWithOptions returns a new Storage with extra options, +// backed by a given `fs.Filesystem` and cache. +func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { + dirOps := dotgit.Options{ + ExclusiveAccess: ops.ExclusiveAccess, + KeepDescriptors: ops.KeepDescriptors, } + dir := dotgit.NewWithOptions(fs, dirOps) return &Storage{ fs: fs, dir: dir, - ObjectStorage: o, + ObjectStorage: ObjectStorage{ + options: ops, + deltaBaseCache: cache, + dir: dir, + }, ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, ConfigStorage: ConfigStorage{dir: dir}, ModuleStorage: ModuleStorage{dir: dir}, - }, nil + } } // Filesystem returns the underlying filesystem @@ -48,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem { return s.fs } +// Init initializes .git directory func (s *Storage) Init() error { return s.dir.Initialize() } diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/memory/storage.go b/vendor/gopkg.in/src-d/go-git.v4/storage/memory/storage.go index 2e325090..6e117424 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/memory/storage.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/memory/storage.go @@ -122,6 +122,16 @@ func (o *ObjectStorage) HasEncodedObject(h plumbing.Hash) (err error) { return nil } +func (o *ObjectStorage) EncodedObjectSize(h plumbing.Hash) ( + size int64, err error) { + obj, ok := o.Objects[h] + if !ok { + return 0, plumbing.ErrObjectNotFound + } + + return obj.Size(), nil +} + func (o *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { obj, ok := o.Objects[h] if !ok || (plumbing.AnyObject != t && obj.Type() != t) { diff --git a/vendor/gopkg.in/src-d/go-git.v4/utils/diff/diff.go b/vendor/gopkg.in/src-d/go-git.v4/utils/diff/diff.go index b840ad6d..f49ae55b 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/utils/diff/diff.go +++ b/vendor/gopkg.in/src-d/go-git.v4/utils/diff/diff.go @@ -16,8 +16,8 @@ import ( // string into the dst string. func Do(src, dst string) (diffs []diffmatchpatch.Diff) { dmp := diffmatchpatch.New() - wSrc, wDst, warray := dmp.DiffLinesToChars(src, dst) - diffs = dmp.DiffMain(wSrc, wDst, false) + wSrc, wDst, warray := dmp.DiffLinesToRunes(src, dst) + diffs = dmp.DiffMainRunes(wSrc, wDst, false) diffs = dmp.DiffCharsToLines(diffs, warray) return diffs } diff --git a/vendor/gopkg.in/src-d/go-git.v4/utils/merkletrie/difftree.go b/vendor/gopkg.in/src-d/go-git.v4/utils/merkletrie/difftree.go index 22940968..d57ed133 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/utils/merkletrie/difftree.go +++ b/vendor/gopkg.in/src-d/go-git.v4/utils/merkletrie/difftree.go @@ -248,14 +248,29 @@ package merkletrie // h: else of i import ( + "context" + "errors" "fmt" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" ) +var ( + ErrCanceled = errors.New("operation canceled") +) + // DiffTree calculates the list of changes between two merkletries. It // uses the provided hashEqual callback to compare noders. func DiffTree(fromTree, toTree noder.Noder, + hashEqual noder.Equal) (Changes, error) { + return DiffTreeContext(context.Background(), fromTree, toTree, hashEqual) +} + +// DiffTree calculates the list of changes between two merkletries. It +// uses the provided hashEqual callback to compare noders. +// Error will be returned if context expires +// Provided context must be non nil +func DiffTreeContext(ctx context.Context, fromTree, toTree noder.Noder, hashEqual noder.Equal) (Changes, error) { ret := NewChanges() @@ -265,6 +280,12 @@ func DiffTree(fromTree, toTree noder.Noder, } for { + select { + case <-ctx.Done(): + return nil, ErrCanceled + default: + } + from := ii.from.current to := ii.to.current diff --git a/vendor/gopkg.in/src-d/go-git.v4/worktree.go b/vendor/gopkg.in/src-d/go-git.v4/worktree.go index 99b2cd12..e45d8154 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/worktree.go +++ b/vendor/gopkg.in/src-d/go-git.v4/worktree.go @@ -713,29 +713,54 @@ func (w *Worktree) readGitmodulesFile() (*config.Modules, error) { } // Clean the worktree by removing untracked files. +// An empty dir could be removed - this is what `git clean -f -d .` does. func (w *Worktree) Clean(opts *CleanOptions) error { s, err := w.Status() if err != nil { return err } - // Check Worktree status to be Untracked, obtain absolute path and delete. - for relativePath, status := range s { - // Check if the path contains a directory and if Dir options is false, - // skip the path. - if relativePath != filepath.Base(relativePath) && !opts.Dir { + root := "" + files, err := w.Filesystem.ReadDir(root) + if err != nil { + return err + } + return w.doClean(s, opts, root, files) +} + +func (w *Worktree) doClean(status Status, opts *CleanOptions, dir string, files []os.FileInfo) error { + for _, fi := range files { + if fi.Name() == ".git" { continue } - // Remove the file only if it's an untracked file. - if status.Worktree == Untracked { - absPath := filepath.Join(w.Filesystem.Root(), relativePath) - if err := os.Remove(absPath); err != nil { + // relative path under the root + path := filepath.Join(dir, fi.Name()) + if fi.IsDir() { + if !opts.Dir { + continue + } + + subfiles, err := w.Filesystem.ReadDir(path) + if err != nil { + return err + } + err = w.doClean(status, opts, path, subfiles) + if err != nil { return err } + } else { + if status.IsUntracked(path) { + if err := w.Filesystem.Remove(path); err != nil { + return err + } + } } } + if opts.Dir { + return doCleanDirectories(w.Filesystem, dir) + } return nil } @@ -881,15 +906,18 @@ func rmFileAndDirIfEmpty(fs billy.Filesystem, name string) error { return err } - path := filepath.Dir(name) - files, err := fs.ReadDir(path) + dir := filepath.Dir(name) + return doCleanDirectories(fs, dir) +} + +// doCleanDirectories removes empty subdirs (without files) +func doCleanDirectories(fs billy.Filesystem, dir string) error { + files, err := fs.ReadDir(dir) if err != nil { return err } - if len(files) == 0 { - fs.Remove(path) + return fs.Remove(dir) } - return nil } diff --git a/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go b/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go index 5fa63ab0..673eb167 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go +++ b/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go @@ -1,9 +1,12 @@ package git import ( + "bytes" "path" + "sort" "strings" + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/index" @@ -92,6 +95,14 @@ func (w *Worktree) buildCommitObject(msg string, opts *CommitOptions, tree plumb ParentHashes: opts.Parents, } + if opts.SignKey != nil { + sig, err := w.buildCommitSignature(commit, opts.SignKey) + if err != nil { + return plumbing.ZeroHash, err + } + commit.PGPSignature = sig + } + obj := w.r.Storer.NewEncodedObject() if err := commit.Encode(obj); err != nil { return plumbing.ZeroHash, err @@ -99,6 +110,22 @@ func (w *Worktree) buildCommitObject(msg string, opts *CommitOptions, tree plumb return w.r.Storer.SetEncodedObject(obj) } +func (w *Worktree) buildCommitSignature(commit *object.Commit, signKey *openpgp.Entity) (string, error) { + encoded := &plumbing.MemoryObject{} + if err := commit.Encode(encoded); err != nil { + return "", err + } + r, err := encoded.Reader() + if err != nil { + return "", err + } + var b bytes.Buffer + if err := openpgp.ArmoredDetachSign(&b, signKey, r, nil); err != nil { + return "", err + } + return b.String(), nil +} + // buildTreeHelper converts a given index.Index file into multiple git objects // reading the blobs from the given filesystem and creating the trees from the // index structure. The created objects are pushed to a given Storer. @@ -162,7 +189,20 @@ func (h *buildTreeHelper) doBuildTree(e *index.Entry, parent, fullpath string) { h.trees[parent].Entries = append(h.trees[parent].Entries, te) } +type sortableEntries []object.TreeEntry + +func (sortableEntries) sortName(te object.TreeEntry) string { + if te.Mode == filemode.Dir { + return te.Name + "/" + } + return te.Name +} +func (se sortableEntries) Len() int { return len(se) } +func (se sortableEntries) Less(i int, j int) bool { return se.sortName(se[i]) < se.sortName(se[j]) } +func (se sortableEntries) Swap(i int, j int) { se[i], se[j] = se[j], se[i] } + func (h *buildTreeHelper) copyTreeToStorageRecursive(parent string, t *object.Tree) (plumbing.Hash, error) { + sort.Sort(sortableEntries(t.Entries)) for i, e := range t.Entries { if e.Mode != filemode.Dir && !e.Hash.IsZero() { continue From 74aa7a132d730e4519e56ee649a948a702d41b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Wed, 17 Oct 2018 13:04:47 +0200 Subject: [PATCH 2/4] Update git storage init for new go-git version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Martín --- dummy/dummy_test.go | 4 ++-- service/git/examples_test.go | 6 ++---- service/git/library.go | 5 ++++- service/git/scanner_test.go | 4 ++-- service/git/service_test.go | 7 +++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dummy/dummy_test.go b/dummy/dummy_test.go index 4c9f6388..b06a1611 100644 --- a/dummy/dummy_test.go +++ b/dummy/dummy_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/suite" "google.golang.org/grpc" "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" log "gopkg.in/src-d/go-log.v1" ) @@ -42,8 +43,7 @@ func (s *DummySuite) SetupSuite() { fixture := fixtures.Basic().One() s.Basic = fixture fs := fixture.DotGit() - sto, err := filesystem.NewStorage(fs) - require.NoError(err) + sto := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) s.apiServer = grpc.NewServer() server := &lookout.DataServerHandler{ diff --git a/service/git/examples_test.go b/service/git/examples_test.go index de354b59..33a8eadc 100644 --- a/service/git/examples_test.go +++ b/service/git/examples_test.go @@ -7,6 +7,7 @@ import ( "github.com/src-d/lookout" "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -18,10 +19,7 @@ func Example() { fixture := fixtures.Basic().One() fs := fixture.DotGit() - storer, err := filesystem.NewStorage(fs) - if err != nil { - panic(err) - } + storer := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) // Create the git service with a repository loader that allows it to find // a repository by ID. diff --git a/service/git/library.go b/service/git/library.go index e822d961..c13f683e 100644 --- a/service/git/library.go +++ b/service/git/library.go @@ -14,6 +14,7 @@ import ( "gopkg.in/src-d/go-errors.v1" "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/config" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -142,7 +143,9 @@ func (l *Library) repositoryStorer(url *lookout.RepositoryInfo) ( return nil, err } - return filesystem.NewStorage(fs) + // TODO(carlosms) take cache size from config + cache := cache.NewObjectLRU(cache.DefaultMaxSize) + return filesystem.NewStorage(fs, cache), nil } func (l *Library) repositoryPath(url *lookout.RepositoryInfo) string { diff --git a/service/git/scanner_test.go b/service/git/scanner_test.go index 9689887b..520c249b 100644 --- a/service/git/scanner_test.go +++ b/service/git/scanner_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/suite" "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -34,8 +35,7 @@ func (s *ScannerSuite) SetupSuite() { fixture := fixtures.Basic().One() fs := fixture.DotGit() - sto, err := filesystem.NewStorage(fs) - require.NoError(err) + sto := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) s.Basic = fixture s.Storer = sto diff --git a/service/git/service_test.go b/service/git/service_test.go index 65278b18..74e903f0 100644 --- a/service/git/service_test.go +++ b/service/git/service_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" fixtures "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -32,8 +33,7 @@ func (s *ServiceSuite) SetupSuite() { fixture := fixtures.Basic().One() fs := fixture.DotGit() - sto, err := filesystem.NewStorage(fs) - require.NoError(err) + sto := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) s.Basic = fixture s.Storer = sto @@ -67,8 +67,7 @@ func (s *ServiceSuite) TestTreeChangesDeleteFile() { fixture := fixtures.ByURL("https://github.com/src-d/go-git.git").One() fs := fixture.DotGit() - sto, err := filesystem.NewStorage(fs) - require.NoError(err) + sto := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize)) dr := NewService(&StorerCommitLoader{sto}) resp, err := dr.GetChanges(context.TODO(), &lookout.ChangesRequest{ From 7b434d94c11a0aacd57554197101e7b97f2b737d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Fri, 19 Oct 2018 16:05:43 +0200 Subject: [PATCH 3/4] Use github auth for git repository fetching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Martín --- Gopkg.lock | 2 ++ cmd/lookoutd/common.go | 11 ++++++- cmd/lookoutd/serve.go | 10 +++--- cmd/lookoutd/work.go | 10 +++--- provider/github/client.go | 26 +++++++++++++++- provider/github/installations.go | 20 +++++++++++- provider/github/poster.go | 5 +-- provider/github/token_transport.go | 18 +++++++++-- provider/github/watcher_test.go | 4 +-- service/git/syncer.go | 49 ++++++++++++++++++++++++------ service/git/syncer_test.go | 42 ++++++++++++++++++++++++- 11 files changed, 167 insertions(+), 30 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 38fb32b9..97345a24 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -818,8 +818,10 @@ "gopkg.in/src-d/go-git.v4", "gopkg.in/src-d/go-git.v4/config", "gopkg.in/src-d/go-git.v4/plumbing", + "gopkg.in/src-d/go-git.v4/plumbing/cache", "gopkg.in/src-d/go-git.v4/plumbing/object", "gopkg.in/src-d/go-git.v4/plumbing/storer", + "gopkg.in/src-d/go-git.v4/plumbing/transport", "gopkg.in/src-d/go-git.v4/plumbing/transport/http", "gopkg.in/src-d/go-git.v4/storage", "gopkg.in/src-d/go-git.v4/storage/filesystem", diff --git a/cmd/lookoutd/common.go b/cmd/lookoutd/common.go index e7f32066..09afe13c 100644 --- a/cmd/lookoutd/common.go +++ b/cmd/lookoutd/common.go @@ -326,8 +326,17 @@ func (c *queueConsumerCommand) initDataHandler() (*lookout.DataServerHandler, er return nil, err } + var authProvider git.AuthProvider + if c.Provider == github.Provider { + if c.pool == nil { + return nil, fmt.Errorf("pool must be initialized with initProvider") + } + + authProvider = c.pool + } + lib := git.NewLibrary(osfs.New(c.Library)) - sync := git.NewSyncer(lib) + sync := git.NewSyncer(lib, authProvider) loader := git.NewLibraryCommitLoader(lib, sync) gitService := git.NewService(loader) diff --git a/cmd/lookoutd/serve.go b/cmd/lookoutd/serve.go index 157991e3..2a4d9bed 100644 --- a/cmd/lookoutd/serve.go +++ b/cmd/lookoutd/serve.go @@ -28,6 +28,11 @@ func (c *ServeCommand) Execute(args []string) error { return err } + err = c.initProvider(conf) + if err != nil { + return err + } + dataHandler, err := c.initDataHandler() if err != nil { return err @@ -49,11 +54,6 @@ func (c *ServeCommand) Execute(args []string) error { return err } - err = c.initProvider(conf) - if err != nil { - return err - } - poster, err := c.initPoster(conf) if err != nil { return err diff --git a/cmd/lookoutd/work.go b/cmd/lookoutd/work.go index 5c33b25b..2bbb0be7 100644 --- a/cmd/lookoutd/work.go +++ b/cmd/lookoutd/work.go @@ -28,6 +28,11 @@ func (c *WorkCommand) Execute(args []string) error { return err } + err = c.initProvider(conf) + if err != nil { + return err + } + dataHandler, err := c.initDataHandler() if err != nil { return err @@ -49,11 +54,6 @@ func (c *WorkCommand) Execute(args []string) error { return err } - err = c.initProvider(conf) - if err != nil { - return err - } - poster, err := c.initPoster(conf) if err != nil { return err diff --git a/provider/github/client.go b/provider/github/client.go index e3d49c14..8a314380 100644 --- a/provider/github/client.go +++ b/provider/github/client.go @@ -1,6 +1,7 @@ package github import ( + "context" "net/http" "strconv" "strings" @@ -8,11 +9,13 @@ import ( "time" "github.com/src-d/lookout" + "github.com/src-d/lookout/service/git" "github.com/src-d/lookout/util/cache" "github.com/src-d/lookout/util/ctxlog" "github.com/google/go-github/github" "github.com/gregjones/httpcache" + "gopkg.in/src-d/go-git.v4/plumbing/transport" log "gopkg.in/src-d/go-log.v1" ) @@ -214,16 +217,36 @@ func (p *ClientPool) notify(e ClientPoolEvent) { } } +var _ git.AuthProvider = &ClientPool{} + +// GitAuth returns a go-git auth method for a repo +func (p *ClientPool) GitAuth(ctx context.Context, repoInfo *lookout.RepositoryInfo) transport.AuthMethod { + c, ok := p.Client(repoInfo.Username, repoInfo.Name) + if !ok { + return nil + } + + return c.gitAuth(ctx) +} + +type gitAuthFn = func(ctx context.Context) transport.AuthMethod + // Client is a wrapper for github.Client that supports cache and provides rate limit information type Client struct { *github.Client cache *cache.ValidableCache limitRT *limitRoundTripper watchMinInterval time.Duration + gitAuth gitAuthFn } // NewClient creates new Client -func NewClient(t http.RoundTripper, cache *cache.ValidableCache, watchMinInterval string) *Client { +func NewClient( + t http.RoundTripper, + cache *cache.ValidableCache, + watchMinInterval string, + gitAuth gitAuthFn, +) *Client { limitRT := &limitRoundTripper{ Base: t, } @@ -247,6 +270,7 @@ func NewClient(t http.RoundTripper, cache *cache.ValidableCache, watchMinInterva cache: cache, limitRT: limitRT, watchMinInterval: interval, + gitAuth: gitAuth, } } diff --git a/provider/github/installations.go b/provider/github/installations.go index a3826f8d..6ee84221 100644 --- a/provider/github/installations.go +++ b/provider/github/installations.go @@ -8,7 +8,10 @@ import ( "github.com/google/go-github/github" "github.com/src-d/lookout" "github.com/src-d/lookout/util/cache" + "github.com/src-d/lookout/util/ctxlog" vcsurl "gopkg.in/sourcegraph/go-vcsurl.v1" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" log "gopkg.in/src-d/go-log.v1" ) @@ -123,9 +126,24 @@ func (t *Installations) createClient(installationID int64) (*Client, error) { return nil, err } + // Auth must be: https://x-access-token:@github.com/owner/repo.git + // Reference: https://developer.github.com/apps/building-github-apps/authenticating-with-github-apps/#http-based-git-access-by-an-installation + gitAuth := func(ctx context.Context) transport.AuthMethod { + token, err := itr.Token() + if err != nil { + ctxlog.Get(ctx).Errorf(err, "failed to get an installation access token") + return nil + } + + return &githttp.BasicAuth{ + Username: "x-access-token", + Password: token, + } + } + // TODO (carlosms): hardcoded, take from config watchMinInterval := "" - return NewClient(itr, t.cache, watchMinInterval), nil + return NewClient(itr, t.cache, watchMinInterval, gitAuth), nil } func (t *Installations) getRepos(iClient *Client) ([]*lookout.RepositoryInfo, error) { diff --git a/provider/github/poster.go b/provider/github/poster.go index 3e38aa6a..05933b92 100644 --- a/provider/github/poster.go +++ b/provider/github/poster.go @@ -3,6 +3,7 @@ package github import ( "context" "fmt" + "reflect" "strings" "github.com/src-d/lookout" @@ -66,7 +67,7 @@ func (p *Poster) Post(ctx context.Context, e lookout.Event, return p.postPR(ctx, ev, aCommentsList) default: - return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type")) + return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type %s", reflect.TypeOf(e))) } } @@ -302,7 +303,7 @@ func (p *Poster) Status(ctx context.Context, e lookout.Event, status lookout.Ana return p.statusPR(ctx, ev, status) default: - return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type")) + return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type %s", reflect.TypeOf(e))) } } diff --git a/provider/github/token_transport.go b/provider/github/token_transport.go index 0261d39b..4ec8e1be 100644 --- a/provider/github/token_transport.go +++ b/provider/github/token_transport.go @@ -1,6 +1,7 @@ package github import ( + "context" "net/http" "github.com/src-d/lookout" @@ -8,6 +9,8 @@ import ( "github.com/src-d/lookout/util/ctxlog" vcsurl "gopkg.in/sourcegraph/go-vcsurl.v1" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" log "gopkg.in/src-d/go-log.v1" ) @@ -42,10 +45,21 @@ func NewClientPoolFromTokens(urlToConfig map[string]ClientConfig, cache *cache.V byClients := make(map[*Client][]*lookout.RepositoryInfo, len(byConfig)) byRepo := make(map[string]*Client, len(urlToConfig)) for conf, repos := range byConfig { - client := NewClient(&roundTripper{ + rt := &roundTripper{ User: conf.User, Password: conf.Token, - }, cache, conf.MinInterval) + } + + // Auth must be: https://@github.com/owner/repo.git + // Reference: https://blog.github.com/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/ + gitAuth := func(ctx context.Context) transport.AuthMethod { + return &githttp.BasicAuth{ + Username: conf.Token, + Password: "", + } + } + + client := NewClient(rt, cache, conf.MinInterval, gitAuth) if _, ok := byClients[client]; !ok { byClients[client] = []*lookout.RepositoryInfo{} diff --git a/provider/github/watcher_test.go b/provider/github/watcher_test.go index ef03114f..3ff01b05 100644 --- a/provider/github/watcher_test.go +++ b/provider/github/watcher_test.go @@ -296,7 +296,7 @@ func (s *WatcherTestSuite) TestCustomMinInterval() { s.mux.HandleFunc("/repos/mock/test/events", eventsHandler(&eventCalls)) clientMinInterval := 200 * time.Millisecond - client := NewClient(nil, s.cache, clientMinInterval.String()) + client := NewClient(nil, s.cache, clientMinInterval.String(), nil) client.BaseURL = s.githubURL client.UploadURL = s.githubURL @@ -483,7 +483,7 @@ func (t *NoopTransport) Get(repo string) http.RoundTripper { } func newClient(githubURL *url.URL, cache *cache.ValidableCache) *Client { - client := NewClient(nil, cache, "") + client := NewClient(nil, cache, "", nil) client.BaseURL = githubURL client.UploadURL = githubURL return client diff --git a/service/git/syncer.go b/service/git/syncer.go index dcb31bbb..f1340ec4 100644 --- a/service/git/syncer.go +++ b/service/git/syncer.go @@ -3,6 +3,7 @@ package git import ( "context" "fmt" + "net/url" "sync" "time" @@ -11,6 +12,8 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/config" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + log "gopkg.in/src-d/go-log.v1" ) @@ -21,11 +24,19 @@ type Syncer struct { m sync.Map // holds mutexes per repository l *Library + + authProvider AuthProvider +} + +// AuthProvider is an object that provides go-git auth methods +type AuthProvider interface { + // GitAuth returns a go-git auth method for a repo + GitAuth(ctx context.Context, repoInfo *lookout.RepositoryInfo) transport.AuthMethod } -// NewSyncer returns a Syncer for the given Library. -func NewSyncer(l *Library) *Syncer { - return &Syncer{l: l} +// NewSyncer returns a Syncer for the given Library. authProvider can be nil +func NewSyncer(l *Library, authProvider AuthProvider) *Syncer { + return &Syncer{l: l, authProvider: authProvider} } // Sync syncs the local git repository to the given reference pointers. @@ -43,7 +54,16 @@ func (s *Syncer) Sync(ctx context.Context, } } - r, err := s.l.GetOrInit(ctx, frp.Repository()) + // TODO(carlosms): Do this scheme change in lookout-sdk, ReferencePointer.Repository() + // GitHub authentication requires the remote to be https:// instead of git:// + repoInfo := frp.Repository() + if repoInfo.RepoHost == "github.com" { + cloneURL, _ := url.Parse(repoInfo.CloneURL) + cloneURL.Scheme = "https" + repoInfo.CloneURL = cloneURL.String() + } + + gitRepo, err := s.l.GetOrInit(ctx, repoInfo) if err != nil { return err } @@ -60,35 +80,44 @@ func (s *Syncer) Sync(ctx context.Context, refspecs = append(refspecs, rs) } - return s.fetch(ctx, frp.InternalRepositoryURL, r, refspecs) + return s.fetch(ctx, repoInfo, gitRepo, refspecs) } -func (s *Syncer) fetch(ctx context.Context, repoURL string, r *git.Repository, refspecs []config.RefSpec) (err error) { - ctxlog.Get(ctx).Infof("fetching references for repository %s: %v", repoURL, refspecs) +func (s *Syncer) fetch(ctx context.Context, repoInfo *lookout.RepositoryInfo, r *git.Repository, refspecs []config.RefSpec) (err error) { + ctxlog.Get(ctx).Infof("fetching references for repository %s: %v", repoInfo.CloneURL, refspecs) start := time.Now() defer func() { if err == nil { ctxlog.Get(ctx). With(log.Fields{"duration": time.Now().Sub(start)}). - Debugf("references %v fetched for repository %s", refspecs, repoURL) + Debugf("references %v fetched for repository %s", refspecs, repoInfo.CloneURL) } // in case of error it will be logged on upper level }() + var auth transport.AuthMethod + if s.authProvider != nil { + auth = s.authProvider.GitAuth(ctx, repoInfo) + } + opts := &git.FetchOptions{ RemoteName: defaultRemoteName, RefSpecs: refspecs, Force: true, + Auth: auth, } - mi, _ := s.m.LoadOrStore(repoURL, &sync.Mutex{}) + mi, _ := s.m.LoadOrStore(repoInfo.CloneURL, &sync.Mutex{}) mutex := mi.(*sync.Mutex) mutex.Lock() defer mutex.Unlock() err = r.FetchContext(ctx, opts) - if err == git.NoErrAlreadyUpToDate { + switch err { + case git.NoErrAlreadyUpToDate: err = nil + case transport.ErrInvalidAuthMethod: + err = fmt.Errorf("wrong go-git authentication method: %s", err.Error()) } return err diff --git a/service/git/syncer_test.go b/service/git/syncer_test.go index e5b279a1..220e7da2 100644 --- a/service/git/syncer_test.go +++ b/service/git/syncer_test.go @@ -9,12 +9,14 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/sourcegraph/go-vcsurl.v1" "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) func TestLibrary_Sync(t *testing.T) { require := require.New(t) library := NewLibrary(memfs.New()) - syncer := NewSyncer(library) + syncer := NewSyncer(library, nil) url, _ := vcsurl.Parse("http://github.com/src-d/lookout") err := syncer.Sync(context.TODO(), lookout.ReferencePointer{ @@ -28,3 +30,41 @@ func TestLibrary_Sync(t *testing.T) { require.NoError(err) require.True(has) } + +var _ AuthProvider = &testAuthProvider{} + +type testAuthProvider struct{} + +var authCalls int + +func (p testAuthProvider) GitAuth(ctx context.Context, repoInfo *lookout.RepositoryInfo) transport.AuthMethod { + authCalls++ + + return &githttp.BasicAuth{ + Username: "", + Password: "", + } +} + +func TestLibrary_Auth(t *testing.T) { + require := require.New(t) + + require.Equal(0, authCalls) + + library := NewLibrary(memfs.New()) + syncer := NewSyncer(library, testAuthProvider{}) + + url, _ := vcsurl.Parse("http://github.com/src-d/lookout") + err := syncer.Sync(context.TODO(), lookout.ReferencePointer{ + InternalRepositoryURL: url.CloneURL, + ReferenceName: "refs/pull/1/head", + Hash: "80a9810a027672a098b07efda3dc305409c9329d", + }) + + require.NoError(err) + has, err := library.Has(url) + require.NoError(err) + require.True(has) + + require.Equal(1, authCalls) +} From ac2f4ed05c2ddeaceacaf474024fdb8ad6d11386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Fri, 19 Oct 2018 17:46:54 +0200 Subject: [PATCH 4/4] Improve log messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Martín --- provider/github/utils.go | 6 +++++- server/server.go | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/provider/github/utils.go b/provider/github/utils.go index f2c11890..33c8c58e 100644 --- a/provider/github/utils.go +++ b/provider/github/utils.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-github/github" "gopkg.in/sourcegraph/go-vcsurl.v1" "gopkg.in/src-d/go-git.v4/plumbing" + log "gopkg.in/src-d/go-log.v1" ) func castEvent(r *lookout.RepositoryInfo, e *github.Event) (lookout.Event, error) { @@ -100,7 +101,10 @@ func castPullRequestBranch(ctx context.Context, b *github.PullRequestBranch) loo r, err := vcsurl.Parse(b.GetRepo().GetCloneURL()) if err != nil { - ctxlog.Get(ctx).Warningf("malformed repository URL on pull request branch") + ctxlog.Get(ctx).With(log.Fields{ + "url": b.GetRepo().GetCloneURL()}, + ).Warningf("malformed repository URL on pull request branch") + return lookout.ReferencePointer{} } diff --git a/server/server.go b/server/server.go index 2602bff9..27239a5b 100644 --- a/server/server.go +++ b/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "reflect" "sync" "github.com/src-d/lookout" @@ -40,7 +41,7 @@ func NewServer(p lookout.Poster, fileGetter lookout.FileGetter, // HandleEvent processes the event calling the analyzers, and posting the results func (s *Server) HandleEvent(ctx context.Context, e lookout.Event) error { ctx, logger := ctxlog.WithLogFields(ctx, log.Fields{ - "event-type": e.Type(), + "event-type": reflect.TypeOf(e), "event-id": e.ID().String(), "repo": e.Revision().Head.InternalRepositoryURL, "head": e.Revision().Head.ReferenceName,