Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support warning in remote targets #1057

Merged
merged 56 commits into from
Aug 23, 2023
Merged
Show file tree
Hide file tree
Changes from 51 commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
eb25c8d
feat: pack OCI image spec v1.1 manifests
qweeah Aug 8, 2023
f8be004
fix e2e
qweeah Aug 8, 2023
125f185
fix line
qweeah Aug 8, 2023
e65ee37
fix e2e
qweeah Aug 8, 2023
e068e8d
use pack type
qweeah Aug 8, 2023
9520375
update e2e go mod
qweeah Aug 8, 2023
84e3b16
feat: support warning in remote targets
qweeah Aug 9, 2023
157c4f9
Merge remote-tracking branch 'origin_src/main' into image-spec/rc4
qweeah Aug 9, 2023
09755f2
remove PackManifestTypeImageV1_1_0_RC2
qweeah Aug 9, 2023
ac14ef8
Merge branch 'image-spec/rc4' into warning
qweeah Aug 9, 2023
e298234
fix artifact push without subject
qweeah Aug 9, 2023
3e03906
add data
qweeah Aug 9, 2023
9e801c5
check returned error
qweeah Aug 9, 2023
a2646a5
Merge remote-tracking branch 'origin_src/main' into image-spec/rc4
qweeah Aug 9, 2023
a71ad66
Merge branch 'image-spec/rc4' into warning
qweeah Aug 9, 2023
93b66b8
default to unknown artifact type when not provided
qweeah Aug 10, 2023
2117e45
fix pack type
qweeah Aug 10, 2023
64b3ab5
fix e2e
qweeah Aug 10, 2023
552f655
fix e2e
qweeah Aug 10, 2023
34da8f6
remove examples
qweeah Aug 10, 2023
4a01645
Merge branch 'main' into image-spec/rc4
qweeah Aug 10, 2023
55b4595
Merge branch 'image-spec/rc4' into warning
qweeah Aug 10, 2023
8c906c2
fix e2e
qweeah Aug 10, 2023
88fd272
fix e2e
qweeah Aug 10, 2023
04dd969
Merge branch 'image-spec/rc4' into warning
qweeah Aug 10, 2023
b4cb666
Merge remote-tracking branch 'origin_src/main' into warning
qweeah Aug 10, 2023
e261f7b
Merge remote-tracking branch 'origin_src/main' into image-spec/rc4
qweeah Aug 10, 2023
098bcbc
fix merging
qweeah Aug 10, 2023
399265d
fix lint
qweeah Aug 10, 2023
5413dd3
fix lint
qweeah Aug 10, 2023
177a19a
bug fix
qweeah Aug 10, 2023
7cc3bf8
add index support
qweeah Aug 10, 2023
788398b
resolve comment
qweeah Aug 10, 2023
93d7a8a
remove duplicated function
qweeah Aug 10, 2023
21f8d41
Merge branch 'image-spec/rc4' into warning
qweeah Aug 11, 2023
c706fb6
Merge remote-tracking branch 'origin_src/main' into warning
qweeah Aug 18, 2023
db9a803
fix `blob delete` and `login`
qweeah Aug 18, 2023
72b5f05
add warning to readonly targets
qweeah Aug 18, 2023
d11b379
deduplicated based on warning value
qweeah Aug 18, 2023
87de8df
udpate options to avoid copylocks error
qweeah Aug 18, 2023
6fcd346
resolve comments
qweeah Aug 21, 2023
aca52ff
fix warning
qweeah Aug 22, 2023
0a044a2
fix windows test error
qweeah Aug 22, 2023
bbb0546
fix error
qweeah Aug 22, 2023
3d3b8a7
add field to logger
qweeah Aug 22, 2023
7b777ea
nit
qweeah Aug 22, 2023
6de7c57
Merge branch 'main' into warning
qweeah Aug 22, 2023
46220d6
fix unit test
qweeah Aug 22, 2023
698f5fb
refactor and make warning depulicated via registry
qweeah Aug 22, 2023
11c46b7
make map a point and init when parsing
qweeah Aug 22, 2023
e8a3a30
deduplicate warning while copying
qweeah Aug 22, 2023
1ff46ff
reorder parameters
qweeah Aug 23, 2023
9a9b387
lazy initialized dedup map
qweeah Aug 23, 2023
8eedab4
update target parsing
qweeah Aug 23, 2023
9f62b98
revert
qweeah Aug 23, 2023
be42427
code clean
qweeah Aug 23, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions cmd/oras/internal/option/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@ import (
"os"
"strconv"
"strings"
"sync"

credentials "github.com/oras-project/oras-credentials-go"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
Expand All @@ -53,6 +55,7 @@ type Remote struct {
distributionSpec distributionSpec
headerFlags []string
headers http.Header
warned *sync.Map
qweeah marked this conversation as resolved.
Show resolved Hide resolved
}

// EnableDistributionSpecFlag set distribution specification flag as applicable.
Expand All @@ -64,6 +67,7 @@ func (opts *Remote) EnableDistributionSpecFlag() {
func (opts *Remote) ApplyFlags(fs *pflag.FlagSet) {
opts.ApplyFlagsWithPrefix(fs, "", "")
fs.BoolVarP(&opts.PasswordFromStdin, "password-stdin", "", false, "read password or identity token from stdin")
opts.warned = &sync.Map{}
qweeah marked this conversation as resolved.
Show resolved Hide resolved
}

func applyPrefix(prefix, description string) (flagPrefix, notePrefix string) {
Expand Down Expand Up @@ -246,33 +250,46 @@ func (opts *Remote) parseCustomHeaders() error {
func (opts *Remote) Credential() auth.Credential {
return credential.Credential(opts.Username, opts.Password)
}
func (opts *Remote) handleWarning(registry string, logger logrus.FieldLogger) func(warning remote.Warning) {
qweeah marked this conversation as resolved.
Show resolved Hide resolved
logger = logger.WithField("registry", registry)
return func(warning remote.Warning) {
if _, loaded := opts.warned.LoadOrStore(struct {
string
remote.WarningValue
}{registry, warning.WarningValue}, true); !loaded {
logger.Warn(warning.Text)
}
}
}

// NewRegistry assembles a oras remote registry.
func (opts *Remote) NewRegistry(hostname string, common Common) (reg *remote.Registry, err error) {
reg, err = remote.NewRegistry(hostname)
func (opts *Remote) NewRegistry(registry string, logger logrus.FieldLogger, common Common) (reg *remote.Registry, err error) {
reg, err = remote.NewRegistry(registry)
if err != nil {
return nil, err
}
hostname = reg.Reference.Registry
reg.PlainHTTP = opts.isPlainHttp(hostname)
if reg.Client, err = opts.authClient(hostname, common.Debug); err != nil {
registry = reg.Reference.Registry
reg.PlainHTTP = opts.isPlainHttp(registry)
reg.HandleWarning = opts.handleWarning(registry, logger)
if reg.Client, err = opts.authClient(registry, common.Debug); err != nil {
return nil, err
}
return
}

// NewRepository assembles a oras remote repository.
func (opts *Remote) NewRepository(reference string, common Common) (repo *remote.Repository, err error) {
func (opts *Remote) NewRepository(reference string, logger logrus.FieldLogger, common Common) (repo *remote.Repository, err error) {
repo, err = remote.NewRepository(reference)
if err != nil {
return nil, err
}
hostname := repo.Reference.Registry
repo.PlainHTTP = opts.isPlainHttp(hostname)
registry := repo.Reference.Registry
repo.PlainHTTP = opts.isPlainHttp(registry)
repo.SkipReferrersGC = true
if repo.Client, err = opts.authClient(hostname, common.Debug); err != nil {
if repo.Client, err = opts.authClient(registry, common.Debug); err != nil {
return nil, err
}
repo.HandleWarning = opts.handleWarning(registry, logger)
if opts.distributionSpec.referrersAPI != nil {
if err := repo.SetReferrersCapability(*opts.distributionSpec.referrersAPI); err != nil {
return nil, err
Expand Down
7 changes: 4 additions & 3 deletions cmd/oras/internal/option/remote_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"reflect"
"testing"

"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
"oras.land/oras-go/v2/registry/remote/auth"
)
Expand Down Expand Up @@ -180,7 +181,7 @@ func TestRemote_NewRegistry(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
reg, err := opts.NewRegistry(uri.Host, opts.Common)
reg, err := opts.NewRegistry(uri.Host, logrus.New(), opts.Common)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
Expand Down Expand Up @@ -208,7 +209,7 @@ func TestRemote_NewRepository(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
repo, err := opts.NewRepository(uri.Host+"/"+testRepo, opts.Common)
repo, err := opts.NewRepository(uri.Host+"/"+testRepo, logrus.New(), opts.Common)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
Expand Down Expand Up @@ -255,7 +256,7 @@ func TestRemote_NewRepository_Retry(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
repo, err := opts.NewRepository(uri.Host+"/"+testRepo, opts.Common)
repo, err := opts.NewRepository(uri.Host+"/"+testRepo, logrus.New(), opts.Common)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
Expand Down
12 changes: 8 additions & 4 deletions cmd/oras/internal/option/target.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (
"fmt"
"os"
"strings"
"sync"

"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/oci"
Expand Down Expand Up @@ -111,7 +113,7 @@ func parseOCILayoutReference(raw string) (path string, ref string, err error) {
}

// NewTarget generates a new target based on opts.
func (opts *Target) NewTarget(common Common) (oras.GraphTarget, error) {
func (opts *Target) NewTarget(common Common, logger logrus.FieldLogger) (oras.GraphTarget, error) {
switch opts.Type {
case TargetTypeOCILayout:
var err error
Expand All @@ -121,7 +123,7 @@ func (opts *Target) NewTarget(common Common) (oras.GraphTarget, error) {
}
return oci.New(opts.Path)
case TargetTypeRemote:
repo, err := opts.NewRepository(opts.RawReference, common)
repo, err := opts.NewRepository(opts.RawReference, logger, common)
if err != nil {
return nil, err
}
Expand All @@ -142,7 +144,7 @@ type ReadOnlyGraphTagFinderTarget interface {
}

// NewReadonlyTargets generates a new read only target based on opts.
func (opts *Target) NewReadonlyTarget(ctx context.Context, common Common) (ReadOnlyGraphTagFinderTarget, error) {
func (opts *Target) NewReadonlyTarget(ctx context.Context, logger logrus.FieldLogger, common Common) (ReadOnlyGraphTagFinderTarget, error) {
switch opts.Type {
case TargetTypeOCILayout:
var err error
Expand All @@ -159,7 +161,7 @@ func (opts *Target) NewReadonlyTarget(ctx context.Context, common Common) (ReadO
}
return oci.NewFromTar(ctx, opts.Path)
case TargetTypeRemote:
repo, err := opts.NewRepository(opts.RawReference, common)
repo, err := opts.NewRepository(opts.RawReference, logger, common)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -199,6 +201,8 @@ func (opts *BinaryTarget) ApplyFlags(fs *pflag.FlagSet) {
opts.From.ApplyFlagsWithPrefix(fs, "from", "source")
opts.To.ApplyFlagsWithPrefix(fs, "to", "destination")
fs.StringArrayVarP(&opts.resolveFlag, "resolve", "", nil, "base DNS rules formatted in `host:port:address[:address_port]` for --from-resolve and --to-resolve")
opts.From.warned = &sync.Map{}
opts.To.warned = opts.From.warned
}

// Parse parses user-provided flags and arguments into option struct.
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/attach.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder
}

func runAttach(ctx context.Context, opts attachOptions) error {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)
annotations, err := opts.LoadManifestAnnotations()
if err != nil {
return err
Expand All @@ -109,7 +109,7 @@ func runAttach(ctx context.Context, opts attachOptions) error {
}
defer store.Close()

dst, err := opts.NewTarget(opts.Common)
dst, err := opts.NewTarget(opts.Common, logger)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/blob/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ Example - Delete a blob and print its descriptor:
}

func deleteBlob(ctx context.Context, opts deleteBlobOptions) (err error) {
ctx, _ = opts.WithContext(ctx)
repo, err := opts.NewRepository(opts.targetRef, opts.Common)
ctx, logger := opts.WithContext(ctx)
repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/blob/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar'
}

func fetchBlob(ctx context.Context, opts fetchBlobOptions) (fetchErr error) {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)
var target oras.ReadOnlyTarget
target, err := opts.NewReadonlyTarget(ctx, opts.Common)
target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/blob/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir':
}

func pushBlob(ctx context.Context, opts pushBlobOptions) (err error) {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)

repo, err := opts.NewTarget(opts.Common)
repo, err := opts.NewTarget(opts.Common, logger)
if err != nil {
return err
}
Expand Down
6 changes: 3 additions & 3 deletions cmd/oras/root/cp.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,10 @@ Example - Copy an artifact with multiple tags with concurrency tuned:
}

func runCopy(ctx context.Context, opts copyOptions) error {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)

// Prepare source
src, err := opts.From.NewReadonlyTarget(ctx, opts.Common)
src, err := opts.From.NewReadonlyTarget(ctx, logger, opts.Common)
if err != nil {
return err
}
Expand All @@ -109,7 +109,7 @@ func runCopy(ctx context.Context, opts copyOptions) error {
}

// Prepare destination
dst, err := opts.To.NewTarget(opts.Common)
dst, err := opts.To.NewTarget(opts.Common, logger)
if err != nil {
return err
}
Expand Down
14 changes: 7 additions & 7 deletions cmd/oras/root/discover.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout
}

func runDiscover(ctx context.Context, opts discoverOptions) error {
ctx, _ = opts.WithContext(ctx)
repo, err := opts.NewReadonlyTarget(ctx, opts.Common)
ctx, logger := opts.WithContext(ctx)
repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common)
if err != nil {
return err
}
Expand All @@ -110,7 +110,7 @@ func runDiscover(ctx context.Context, opts discoverOptions) error {

if opts.outputType == "tree" {
root := tree.New(fmt.Sprintf("%s@%s", opts.Path, desc.Digest))
err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, &opts)
err = opts.fetchAllReferrers(ctx, repo, desc, opts.artifactType, root)
if err != nil {
return err
}
Expand Down Expand Up @@ -138,7 +138,7 @@ func runDiscover(ctx context.Context, opts discoverOptions) error {
return nil
}

func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error {
func (opts *discoverOptions) fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node) error {
results, err := graph.Referrers(ctx, repo, desc, artifactType)
if err != nil {
return err
Expand All @@ -156,14 +156,14 @@ func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc
referrerNode.AddPath(strings.TrimSpace(string(bytes)))
}
}
err := fetchAllReferrers(
err := opts.fetchAllReferrers(
ctx, repo,
ocispec.Descriptor{
Digest: r.Digest,
Size: r.Size,
MediaType: r.MediaType,
},
artifactType, referrerNode, opts)
artifactType, referrerNode)
if err != nil {
return err
}
Expand All @@ -189,7 +189,7 @@ func printDiscoveredReferrersTable(refs []ocispec.Descriptor, verbose bool) erro
print(ref.ArtifactType, ref.Digest)
if verbose {
if err := printJSON(ref); err != nil {
return fmt.Errorf("Error printing JSON: %w", err)
return fmt.Errorf("error printing JSON: %w", err)
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/login.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ Example - Log in with username and password in an interactive terminal and no TL
}

func runLogin(ctx context.Context, opts loginOptions) (err error) {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)

// prompt for credential
if opts.Password == "" {
Expand Down Expand Up @@ -108,7 +108,7 @@ func runLogin(ctx context.Context, opts loginOptions) (err error) {
if err != nil {
return err
}
remote, err := opts.Remote.NewRegistry(opts.Hostname, opts.Common)
remote, err := opts.Remote.NewRegistry(opts.Hostname, logger, opts.Common)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/manifest/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614
}

func deleteManifest(ctx context.Context, opts deleteOptions) error {
ctx, _ = opts.WithContext(ctx)
repo, err := opts.NewRepository(opts.targetRef, opts.Common)
ctx, logger := opts.WithContext(ctx)
repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/manifest/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar':
}

func fetchManifest(ctx context.Context, opts fetchOptions) (fetchErr error) {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)

target, err := opts.NewReadonlyTarget(ctx, opts.Common)
target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/manifest/fetch_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ Example - Fetch and print the prettified descriptor of the config:
}

func fetchConfig(ctx context.Context, opts fetchConfigOptions) (fetchErr error) {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)

repo, err := opts.NewReadonlyTarget(ctx, opts.Common)
repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/manifest/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,10 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit
}

func pushManifest(ctx context.Context, opts pushOptions) error {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)
var target oras.Target
var err error
target, err = opts.NewTarget(opts.Common)
target, err = opts.NewTarget(opts.Common, logger)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/oras/root/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ Example - Pull artifact files from an OCI layout archive 'layout.tar':
}

func runPull(ctx context.Context, opts pullOptions) error {
ctx, _ = opts.WithContext(ctx)
ctx, logger := opts.WithContext(ctx)
// Copy Options
var printed sync.Map
copyOptions := oras.DefaultCopyOptions
Expand Down Expand Up @@ -182,7 +182,7 @@ func runPull(ctx context.Context, opts pullOptions) error {
return ret, nil
}

target, err := opts.NewReadonlyTarget(ctx, opts.Common)
target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common)
if err != nil {
return err
}
Expand Down
Loading
Loading