Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(dl): add credential support for dl service #163

Merged
merged 1 commit into from
Jul 23, 2024
Merged

feat(dl): add credential support for dl service #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dl/cmd/server/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func main() {
secureF = flag.Bool("secure", false, "Use secure scheme (https or grpcs)")
dbgF = flag.Bool("debug", false, "Log request and response bodies")
ks3CfgPathF = flag.String("ks3-config", "ks3.yaml", "ks3 config yaml file path")
ociCfgPathF = flag.String("oci-config", "oci.yaml", "oci config yaml file path")
)
flag.Parse()

Expand All @@ -47,7 +48,7 @@ func main() {
)
{
healthSvc = dl.NewHealth(logger)
ociSvc = dl.NewOci(logger)
ociSvc = dl.NewOci(logger, ociCfgPathF)
ks3Svc = dl.NewKs3(logger, *ks3CfgPathF)
}

Expand Down
5 changes: 4 additions & 1 deletion dl/ks3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ type ks3srvc struct {
}

func newKS3Client(cfg *pkgks3.Config) *s3.S3 {
var cre = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "")
var cre *credentials.Credentials
if cfg != nil && cfg.AccessKey != "" && cfg.SecretKey != "" {
cre = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "")
}
awsConfig := aws.Config{
Region: cfg.Region, // Ref: https://docs.ksyun.com/documents/6761
Credentials: cre,
Expand Down
63 changes: 57 additions & 6 deletions dl/oci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,40 +5,87 @@ import (
"io"
"log"
"net/url"
"os"
"strings"

oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci"
pkgoci "github.com/PingCAP-QE/ee-apps/dl/pkg/oci"
"gopkg.in/yaml.v3"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
"oras.land/oras-go/v2/registry/remote/retry"
)

// oci service example implementation.
// The example methods log the requests and return zero values.
type ocisrvc struct {
logger *log.Logger
logger *log.Logger
credential *auth.Credential
}

// NewOci returns the oci service implementation.
func NewOci(logger *log.Logger) oci.Service {
return &ocisrvc{logger}
func NewOci(logger *log.Logger, cfgFile *string) oci.Service {
var cfg pkgoci.Config
if cfgFile == nil {
return &ocisrvc{logger: logger, credential: &auth.EmptyCredential}
}

cfgBytes, err := os.ReadFile(*cfgFile)
if err != nil {
logger.Fatalf("Failed to load configuration: %v", err)
}
if err := yaml.Unmarshal(cfgBytes, &cfg); err != nil {
logger.Fatalf("Failed to load configuration: %v", err)
}

return &ocisrvc{logger: logger, credential: &auth.Credential{
Username: cfg.Username,
Password: cfg.Password,
}}
}

// ListFiles implements list-files.
func (s *ocisrvc) ListFiles(ctx context.Context, p *oci.ListFilesPayload) (res []string, err error) {
s.logger.Print("oci.list-files")

files, err := pkgoci.ListFiles(ctx, p.Repository, p.Tag)
repository, err := s.getTargetRepo(p.Repository)
if err != nil {
return nil, err
}

files, err := pkgoci.ListFiles(ctx, repository, p.Tag)
if err != nil {
return nil, oci.MakeInvalidFilePath(err)
}

return files, nil
}

func (s *ocisrvc) getTargetRepo(repo string) (*remote.Repository, error) {
repository, err := remote.NewRepository(repo)
if err != nil {
return nil, err
}

reg := strings.SplitN(repo, "/", 2)[0]
repository.Client = &auth.Client{
Client: retry.DefaultClient,
Cache: auth.DefaultCache,
Credential: auth.StaticCredential(reg, *s.credential),
}

return repository, nil
}

// DownloadFile implements download-file.
func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload) (res *oci.DownloadFileResult, resp io.ReadCloser, err error) {
s.logger.Print("oci.download-files")

rc, length, err := pkgoci.NewFileReadCloser(ctx, p.Repository, p.Tag, p.File)
repository, err := s.getTargetRepo(p.Repository)
if err != nil {
return nil, nil, err
}
rc, length, err := pkgoci.NewFileReadCloser(ctx, repository, p.Tag, p.File)
if err != nil {
return nil, nil, err
}
Expand All @@ -54,7 +101,11 @@ func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload)
func (s *ocisrvc) DownloadFileSha256(ctx context.Context, p *oci.DownloadFileSha256Payload) (res *oci.DownloadFileSha256Result, resp io.ReadCloser, err error) {
s.logger.Print("oci.download-file-sha256")

value, err := pkgoci.GetFileSHA256(ctx, p.Repository, p.Tag, p.File)
repository, err := s.getTargetRepo(p.Repository)
if err != nil {
return nil, nil, err
}
value, err := pkgoci.GetFileSHA256(ctx, repository, p.Tag, p.File)
if err != nil {
return nil, nil, err
}
Expand Down
6 changes: 6 additions & 0 deletions dl/pkg/oci/cfg.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package oci

type Config struct {
Username string `yaml:"username,omitempty" json:"username,omitempty"`
Password string `yaml:"password,omitempty" json:"password,omitempty"`
}
35 changes: 4 additions & 31 deletions dl/pkg/oci/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,14 @@ import (
"fmt"
"io"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
oras "oras.land/oras-go/v2"
"oras.land/oras-go/v2/registry/remote"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

const AnnotationKeyFileName = "org.opencontainers.image.title"

func ListFiles(ctx context.Context, repo, tag string) ([]string, error) {
repository, err := remote.NewRepository(repo)
if err != nil {
return nil, err
}

// Note: The below code can be omitted if authentication is not required
// reg := strings.SplitN(repo, "/", 2)[0]
// repository.Client = &auth.Client{
// Client: retry.DefaultClient,
// Cache: auth.DefaultCache,
// Credential: auth.StaticCredential(reg, auth.Credential{
// Username: "username",
// Password: "password",
// }),
// }

func ListFiles(ctx context.Context, repository *remote.Repository, tag string) ([]string, error) {
layers, err := listArtifactLayers(ctx, repository, tag)
if err != nil {
return nil, err
Expand All @@ -44,12 +27,7 @@ func ListFiles(ctx context.Context, repo, tag string) ([]string, error) {
return ret, nil
}

func NewFileReadCloser(ctx context.Context, repo, tag, filename string) (io.ReadCloser, int64, error) {
repository, err := remote.NewRepository(repo)
if err != nil {
return nil, 0, err
}

func NewFileReadCloser(ctx context.Context, repository *remote.Repository, tag, filename string) (io.ReadCloser, int64, error) {
// 1. get desired file descriptor in the artifact.
// destination := strings.Join([]string{repo, tag}, ":")
desiredFileDescriptor, err := fetchFileDescriptor(ctx, repository, tag, filename)
Expand All @@ -67,12 +45,7 @@ func NewFileReadCloser(ctx context.Context, repo, tag, filename string) (io.Read
return rc, desiredFileDescriptor.Size, nil
}

func GetFileSHA256(ctx context.Context, repo, tag, filename string) (string, error) {
repository, err := remote.NewRepository(repo)
if err != nil {
return "", err
}

func GetFileSHA256(ctx context.Context, repository oras.ReadOnlyTarget, tag, filename string) (string, error) {
// 1. get desired file descriptor in the artifact.
// destination := strings.Join([]string{repo, tag}, ":")
desiredFileDescriptor, err := fetchFileDescriptor(ctx, repository, tag, filename)
Expand Down