From 6dcfc9c724c6c1cc4e73904e141d88dbd3a0c182 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 23 Jul 2024 10:19:33 +0800 Subject: [PATCH] feat(dl): add credential support for dl service (#163) Signed-off-by: wuhuizuo Signed-off-by: wuhuizuo --- dl/cmd/server/main.go | 3 +- dl/ks3.go | 5 +++- dl/oci.go | 63 ++++++++++++++++++++++++++++++++++++++---- dl/pkg/oci/cfg.go | 6 ++++ dl/pkg/oci/download.go | 35 +++-------------------- 5 files changed, 73 insertions(+), 39 deletions(-) create mode 100644 dl/pkg/oci/cfg.go diff --git a/dl/cmd/server/main.go b/dl/cmd/server/main.go index ef7580c..7beef04 100644 --- a/dl/cmd/server/main.go +++ b/dl/cmd/server/main.go @@ -28,6 +28,7 @@ func main() { secureF = flag.Bool("secure", false, "Use secure scheme (https or grpcs)") dbgF = flag.Bool("debug", false, "Log request and response bodies") ks3CfgPathF = flag.String("ks3-config", "ks3.yaml", "ks3 config yaml file path") + ociCfgPathF = flag.String("oci-config", "oci.yaml", "oci config yaml file path") ) flag.Parse() @@ -47,7 +48,7 @@ func main() { ) { healthSvc = dl.NewHealth(logger) - ociSvc = dl.NewOci(logger) + ociSvc = dl.NewOci(logger, ociCfgPathF) ks3Svc = dl.NewKs3(logger, *ks3CfgPathF) } diff --git a/dl/ks3.go b/dl/ks3.go index 6d0efda..5bc11e6 100644 --- a/dl/ks3.go +++ b/dl/ks3.go @@ -25,7 +25,10 @@ type ks3srvc struct { } func newKS3Client(cfg *pkgks3.Config) *s3.S3 { - var cre = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "") + var cre *credentials.Credentials + if cfg != nil && cfg.AccessKey != "" && cfg.SecretKey != "" { + cre = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "") + } awsConfig := aws.Config{ Region: cfg.Region, // Ref: https://docs.ksyun.com/documents/6761 Credentials: cre, diff --git a/dl/oci.go b/dl/oci.go index c9a3d94..209ecf3 100644 --- a/dl/oci.go +++ b/dl/oci.go @@ -5,28 +5,55 @@ import ( "io" "log" "net/url" + "os" "strings" oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci" pkgoci "github.com/PingCAP-QE/ee-apps/dl/pkg/oci" + "gopkg.in/yaml.v3" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" ) // oci service example implementation. // The example methods log the requests and return zero values. type ocisrvc struct { - logger *log.Logger + logger *log.Logger + credential *auth.Credential } // NewOci returns the oci service implementation. -func NewOci(logger *log.Logger) oci.Service { - return &ocisrvc{logger} +func NewOci(logger *log.Logger, cfgFile *string) oci.Service { + var cfg pkgoci.Config + if cfgFile == nil { + return &ocisrvc{logger: logger, credential: &auth.EmptyCredential} + } + + cfgBytes, err := os.ReadFile(*cfgFile) + if err != nil { + logger.Fatalf("Failed to load configuration: %v", err) + } + if err := yaml.Unmarshal(cfgBytes, &cfg); err != nil { + logger.Fatalf("Failed to load configuration: %v", err) + } + + return &ocisrvc{logger: logger, credential: &auth.Credential{ + Username: cfg.Username, + Password: cfg.Password, + }} } // ListFiles implements list-files. func (s *ocisrvc) ListFiles(ctx context.Context, p *oci.ListFilesPayload) (res []string, err error) { s.logger.Print("oci.list-files") - files, err := pkgoci.ListFiles(ctx, p.Repository, p.Tag) + repository, err := s.getTargetRepo(p.Repository) + if err != nil { + return nil, err + } + + files, err := pkgoci.ListFiles(ctx, repository, p.Tag) if err != nil { return nil, oci.MakeInvalidFilePath(err) } @@ -34,11 +61,31 @@ func (s *ocisrvc) ListFiles(ctx context.Context, p *oci.ListFilesPayload) (res [ return files, nil } +func (s *ocisrvc) getTargetRepo(repo string) (*remote.Repository, error) { + repository, err := remote.NewRepository(repo) + if err != nil { + return nil, err + } + + reg := strings.SplitN(repo, "/", 2)[0] + repository.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.DefaultCache, + Credential: auth.StaticCredential(reg, *s.credential), + } + + return repository, nil +} + // DownloadFile implements download-file. func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload) (res *oci.DownloadFileResult, resp io.ReadCloser, err error) { s.logger.Print("oci.download-files") - rc, length, err := pkgoci.NewFileReadCloser(ctx, p.Repository, p.Tag, p.File) + repository, err := s.getTargetRepo(p.Repository) + if err != nil { + return nil, nil, err + } + rc, length, err := pkgoci.NewFileReadCloser(ctx, repository, p.Tag, p.File) if err != nil { return nil, nil, err } @@ -54,7 +101,11 @@ func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload) func (s *ocisrvc) DownloadFileSha256(ctx context.Context, p *oci.DownloadFileSha256Payload) (res *oci.DownloadFileSha256Result, resp io.ReadCloser, err error) { s.logger.Print("oci.download-file-sha256") - value, err := pkgoci.GetFileSHA256(ctx, p.Repository, p.Tag, p.File) + repository, err := s.getTargetRepo(p.Repository) + if err != nil { + return nil, nil, err + } + value, err := pkgoci.GetFileSHA256(ctx, repository, p.Tag, p.File) if err != nil { return nil, nil, err } diff --git a/dl/pkg/oci/cfg.go b/dl/pkg/oci/cfg.go new file mode 100644 index 0000000..1d33cc9 --- /dev/null +++ b/dl/pkg/oci/cfg.go @@ -0,0 +1,6 @@ +package oci + +type Config struct { + Username string `yaml:"username,omitempty" json:"username,omitempty"` + Password string `yaml:"password,omitempty" json:"password,omitempty"` +} diff --git a/dl/pkg/oci/download.go b/dl/pkg/oci/download.go index a62b16d..9827337 100644 --- a/dl/pkg/oci/download.go +++ b/dl/pkg/oci/download.go @@ -6,31 +6,14 @@ import ( "fmt" "io" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" oras "oras.land/oras-go/v2" "oras.land/oras-go/v2/registry/remote" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) const AnnotationKeyFileName = "org.opencontainers.image.title" -func ListFiles(ctx context.Context, repo, tag string) ([]string, error) { - repository, err := remote.NewRepository(repo) - if err != nil { - return nil, err - } - - // Note: The below code can be omitted if authentication is not required - // reg := strings.SplitN(repo, "/", 2)[0] - // repository.Client = &auth.Client{ - // Client: retry.DefaultClient, - // Cache: auth.DefaultCache, - // Credential: auth.StaticCredential(reg, auth.Credential{ - // Username: "username", - // Password: "password", - // }), - // } - +func ListFiles(ctx context.Context, repository *remote.Repository, tag string) ([]string, error) { layers, err := listArtifactLayers(ctx, repository, tag) if err != nil { return nil, err @@ -44,12 +27,7 @@ func ListFiles(ctx context.Context, repo, tag string) ([]string, error) { return ret, nil } -func NewFileReadCloser(ctx context.Context, repo, tag, filename string) (io.ReadCloser, int64, error) { - repository, err := remote.NewRepository(repo) - if err != nil { - return nil, 0, err - } - +func NewFileReadCloser(ctx context.Context, repository *remote.Repository, tag, filename string) (io.ReadCloser, int64, error) { // 1. get desired file descriptor in the artifact. // destination := strings.Join([]string{repo, tag}, ":") desiredFileDescriptor, err := fetchFileDescriptor(ctx, repository, tag, filename) @@ -67,12 +45,7 @@ func NewFileReadCloser(ctx context.Context, repo, tag, filename string) (io.Read return rc, desiredFileDescriptor.Size, nil } -func GetFileSHA256(ctx context.Context, repo, tag, filename string) (string, error) { - repository, err := remote.NewRepository(repo) - if err != nil { - return "", err - } - +func GetFileSHA256(ctx context.Context, repository oras.ReadOnlyTarget, tag, filename string) (string, error) { // 1. get desired file descriptor in the artifact. // destination := strings.Join([]string{repo, tag}, ":") desiredFileDescriptor, err := fetchFileDescriptor(ctx, repository, tag, filename)