feature(main): add container storage (#2)

Signed-off-by: cuisongliu <cuisongliu@qq.com>

.github/workflows/go.yml

@@ -34,3 +34,13 @@ jobs:
           cp dist/sreg_linux_amd64_v1/sreg  sreg
           chmod a+x sreg
           ./sreg version
+      - name: Upload linux-amd64
+        uses: actions/upload-artifact@v3
+        with:
+          name: sreg-linux-amd64
+          path: dist/sreg_linux_amd64_v1/sreg
+      - name: Upload linux-arm64
+        uses: actions/upload-artifact@v3
+        with:
+          name: sreg-linux-arm64
+          path: dist/sreg_linux_arm64/sreg

.goreleaser.yml

@@ -7,15 +7,36 @@ builds:
     goarch:
       - amd64
       - arm64
+    flags:
+      - -trimpath
     ldflags:
       - -s -w
       - -X github.com/labring/sreg/pkg/version.gitVersion={{.Version}}
       - -X github.com/labring/sreg/pkg/version.gitCommit={{.ShortCommit}}
       - -X github.com/labring/sreg/pkg/version.buildDate={{.Date}}
     tags:
       - containers_image_openpgp
+      - netgo
       - exclude_graphdriver_devicemapper
-
+      - static
+      - osusergo
+      - exclude_graphdriver_btrfs
+    overrides:
+      - goos: linux
+        goarch: amd64
+        goamd64: v1
+        goarm: ""
+        gomips: ""
+        env:
+          - CGO_ENABLED=1
+          - CC=x86_64-linux-gnu-gcc
+      - goos: linux
+        goarch: arm64
+        goarm: ""
+        gomips: ""
+        env:
+          - CGO_ENABLED=1
+          - CC=aarch64-linux-gnu-gcc
 checksum:
   name_template: 'checksums.txt'
 snapshot:

cmd/root.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"github.com/labring/sreg/pkg/registry/commands"
 	"github.com/labring/sreg/pkg/utils/logger"
+	"github.com/sirupsen/logrus"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -50,6 +51,9 @@ func Execute() {
 func init() {
 	cobra.OnInitialize(func() {
 		logger.CfgConsoleLogger(debug, false)
+		if debug {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
 	})
 	rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debug logger")
 

pkg/buildimage/tar_images.go

@@ -37,6 +37,9 @@ func TarList(dir string) ([]string, error) {
 		return nil, wrapGetImageErr(err, tarDir)
 	}
 	for i, image := range images {
+		if image == "" {
+			continue
+		}
 		parts := strings.SplitN(image, "@", 2)
 		if len(parts) != 2 {
 			return nil, fmt.Errorf("invalid image format: %s", image)

pkg/registry/commands/flags.go

@@ -55,9 +55,12 @@ func (opts *registrySaveResults) CheckAuth() (map[string]types.AuthConfig, error
 type registrySaveRawResults struct {
 	*registrySaveResults
 	images []string
+	tars   []string
 }
 
 func (opts *registrySaveRawResults) RegisterFlags(fs *pflag.FlagSet) {
 	opts.registrySaveResults.RegisterFlags(fs)
 	fs.StringSliceVar(&opts.images, "images", []string{}, "images list")
+	fs.StringSliceVar(&opts.tars, "tars", []string{}, "tar list, eg: --tars=docker-archive:/root/config_main.tar@library/config_main")
+
 }

pkg/registry/commands/save.go

@@ -33,7 +33,7 @@ import (
 
 func NewRegistryImageSaveCmd(examplePrefix string) *cobra.Command {
 	var auth map[string]types.AuthConfig
-	var images []string
+	var images, tars []string
 	flagsResults := registrySaveRawResults{
 		registrySaveResults: new(registrySaveResults),
 	}
@@ -42,43 +42,53 @@ func NewRegistryImageSaveCmd(examplePrefix string) *cobra.Command {
 		Short: "save images to local registry dir",
 		Example: fmt.Sprintf(`
 %[1]s save --registry-dir=/tmp/registry .
+%[1]s save --registry-dir=/tmp/registry --images=containers-storage:docker.io/labring/coredns:v0.0.1
+%[1]s save --registry-dir=/tmp/registry --images=docker-daemon:docker.io/library/nginx:latest
+%[1]s save --registry-dir=/tmp/registry --tars=docker-archive:/root/config_main.tar@library/config_main
+%[1]s save --registry-dir=/tmp/registry --tars=oci-archive:/root/config_main.tar@library/config_main
 %[1]s save --registry-dir=/tmp/registry --images=docker.io/library/busybox:latest`, examplePrefix),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			is := save.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth)
-			outImages, err := is.SaveImages(images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
-			if err != nil {
-				return err
+			if len(images) > 0 {
+				is := save.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth)
+				outImages, err := is.SaveImages(images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
+				if err != nil {
+					return err
+				}
+				logger.Info("images pulled: %+v", outImages)
 			}
-			logger.Info("images pulled: %+v", outImages)
-			if args[0] != "" {
+
+			if len(tars) > 0 {
 				tarIs := save.NewImageTarSaver(context.Background(), flagsResults.registryPullMaxPullProcs)
-				tars, err := buildimage.TarList(args[0])
+				outTars, err := tarIs.SaveImages(tars, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
 				if err != nil {
 					return err
 				}
-				if len(tars) != 0 {
-					outTars, err := tarIs.SaveImages(tars, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
-					if err != nil {
-						return err
-					}
-					logger.Info("images tar saved: %+v", outTars)
-				}
+				logger.Info("images tar saved: %+v", outTars)
 			}
-
 			return nil
 		},
 		PreRunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) == 0 && len(flagsResults.images) == 0 {
-				return errors.New("'--images' and args cannot be empty at the same time")
+			if len(args) == 0 && len(flagsResults.images) == 0 && len(flagsResults.tars) == 0 {
+				return errors.New("'--images' '--tars' and args cannot be empty at the same time")
 			}
 			var err error
-			if len(flagsResults.images) > 0 {
-				images = flagsResults.images
+			if len(args) == 0 {
+				if len(flagsResults.images) > 0 {
+					images = flagsResults.images
+				}
+				if len(flagsResults.tars) > 0 {
+					tars = flagsResults.tars
+				}
 			} else {
 				images, err = buildimage.List(args[0])
-			}
-			if err != nil {
-				return err
+				if err != nil {
+					return err
+				}
+				tars, err = buildimage.TarList(args[0])
+				if err != nil {
+					return err
+				}
+
 			}
 			auth, err = flagsResults.CheckAuth()
 			if err != nil {

pkg/registry/save/registry_save.go

@@ -17,7 +17,6 @@ package save
 import (
 	"context"
 	"fmt"
-	"github.com/containers/image/v5/transports/alltransports"
 	"strings"
 	stdsync "sync"
 	"time"
@@ -75,17 +74,9 @@ func (is *tmpRegistryImage) SaveImages(images []string, dir string, platform v1.
 				mu.Unlock()
 			}()
 			var srcRef itype.ImageReference
-			if strings.HasPrefix(img, "docker-daemon") {
-				logger.Info("Using containers-storage or docker-daemon as image transport")
-				srcRef, err = alltransports.ParseImageName(img)
-				if err != nil {
-					return fmt.Errorf("invalid source name %s: %v", img, err)
-				}
-			} else {
-				srcRef, err = sync.ImageNameToReference(sys, img, is.auths)
-				if err != nil {
-					return err
-				}
+			srcRef, err = sync.ImageNameToReference(sys, img, is.auths)
+			if err != nil {
+				return err
 			}
 			err = sync.RegistryToImage(is.ctx, sys, srcRef, ep, copy.CopySystemImage)
 			if err != nil {

pkg/registry/save/registry_tars_save.go

@@ -87,6 +87,9 @@ func (is *tmpTarRegistryImage) SaveImages(images []string, dir string, platform
 				<-numCh
 				mu.Unlock()
 			}()
+			if strings.TrimSpace(img) == "" {
+				return nil
+			}
 			allImage := strings.Split(img, "@")
 			srcRef, err := alltransports.ParseImageName(allImage[0])
 			if err != nil {

pkg/registry/sync/sync.go

@@ -64,6 +64,12 @@ func ToRegistry(ctx context.Context, opts *Options) error {
 	if err != nil {
 		return err
 	}
+	var allError error
+	defer func() {
+		if err := policyContext.Destroy(); err != nil {
+			allError = fmt.Errorf("error tearing down policy context: %v", err)
+		}
+	}()
 	repos, err := docker.SearchRegistry(ctx, sys, src, "", 1<<10)
 	if err != nil {
 		return err
@@ -115,7 +121,7 @@ func ToRegistry(ctx context.Context, opts *Options) error {
 			}
 		}
 	}
-	return nil
+	return allError
 }
 
 func getRetryOptions() *retry.RetryOptions {
@@ -131,6 +137,27 @@ func getRetryOptions() *retry.RetryOptions {
 }
 
 func ImageNameToReference(sys *types.SystemContext, img string, auth map[string]dtype.AuthConfig) (types.ImageReference, error) {
+	if err := reexecIfNecessaryForImages(img); err != nil {
+		return nil, err
+	}
+	transport := alltransports.TransportFromImageName(img)
+	if transport != nil && transport.Name() == "containers-storage" {
+		logger.Info("Using containers-storage as image transport")
+		srcRef, err := alltransports.ParseImageName(img)
+		if err != nil {
+			return nil, fmt.Errorf("invalid source name %s: %v", img, err)
+		}
+		return srcRef, nil
+	}
+	if transport != nil && transport.Name() == "docker-daemon" {
+		logger.Info("Using docker-daemon as image transport")
+		srcRef, err := alltransports.ParseImageName(img)
+		if err != nil {
+			return nil, fmt.Errorf("invalid source name %s: %v", img, err)
+		}
+		return srcRef, nil
+	}
+
 	src, err := name.ParseReference(img)
 	if err != nil {
 		return nil, fmt.Errorf("ref invalid source name %s: %v", img, err)
@@ -174,15 +201,24 @@ func RegistryToImage(ctx context.Context, sys *types.SystemContext, src types.Im
 	if err != nil {
 		return err
 	}
-	return retry.RetryIfNecessary(ctx, func() error {
+	var allError error
+	defer func() {
+		if err := policyContext.Destroy(); err != nil {
+			allError = fmt.Errorf("error tearing down policy context: %v", err)
+		}
+	}()
+	if err = retry.RetryIfNecessary(ctx, func() error {
 		_, err = copy.Image(ctx, policyContext, destRef, src, &copy.Options{
 			SourceCtx:          sys,
 			DestinationCtx:     sys,
 			ImageListSelection: selection,
 			ReportWriter:       os.Stdout,
 		})
 		return err
-	}, getRetryOptions())
+	}, getRetryOptions()); err != nil {
+		return err
+	}
+	return allError
 }
 
 func ArchiveToImage(ctx context.Context, sys *types.SystemContext, src types.ImageReference, dst string, selection copy.ImageListSelection) error {
@@ -195,15 +231,24 @@ func ArchiveToImage(ctx context.Context, sys *types.SystemContext, src types.Ima
 	if err != nil {
 		return err
 	}
-	return retry.RetryIfNecessary(ctx, func() error {
+	var allError error
+	defer func() {
+		if err = policyContext.Destroy(); err != nil {
+			allError = fmt.Errorf("error tearing down policy context: %v", err)
+		}
+	}()
+	if err = retry.RetryIfNecessary(ctx, func() error {
 		_, err = copy.Image(ctx, policyContext, destRef, src, &copy.Options{
 			SourceCtx:          sys,
 			DestinationCtx:     sys,
 			ImageListSelection: selection,
 			ReportWriter:       os.Stdout,
 		})
 		return err
-	}, getRetryOptions())
+	}, getRetryOptions()); err != nil {
+		return err
+	}
+	return allError
 }
 
 func getPolicyContext() (*signature.PolicyContext, error) {

pkg/registry/sync/unshare.go

@@ -0,0 +1,8 @@
+//go:build !linux
+// +build !linux
+
+package sync
+
+func reexecIfNecessaryForImages(inputImageNames ...string) error {
+	return nil
+}

pkg/registry/sync/unshare_linux.go

@@ -0,0 +1,48 @@
+package sync
+
+import (
+	"github.com/containers/image/v5/transports/alltransports"
+	"github.com/containers/storage/pkg/unshare"
+	"github.com/pkg/errors"
+	"github.com/syndtr/gocapability/capability"
+)
+
+var neededCapabilities = []capability.Cap{
+	capability.CAP_CHOWN,
+	capability.CAP_DAC_OVERRIDE,
+	capability.CAP_FOWNER,
+	capability.CAP_FSETID,
+	capability.CAP_MKNOD,
+	capability.CAP_SETFCAP,
+}
+
+func maybeReexec() error {
+	// With Skopeo we need only the subset of the root capabilities necessary
+	// for pulling an image to the storage.  Do not attempt to create a namespace
+	// if we already have the capabilities we need.
+	capabilities, err := capability.NewPid(0)
+	if err != nil {
+		return errors.Wrapf(err, "error reading the current capabilities sets")
+	}
+	for _, cap := range neededCapabilities {
+		if !capabilities.Get(capability.EFFECTIVE, cap) {
+			// We miss a capability we need, create a user namespaces
+			unshare.MaybeReexecUsingUserNamespace(true)
+			return nil
+		}
+	}
+	return nil
+}
+
+func reexecIfNecessaryForImages(imageNames ...string) error {
+	// Check if container-storage is used before doing unshare
+	for _, imageName := range imageNames {
+		transport := alltransports.TransportFromImageName(imageName)
+		// Hard-code the storage name to avoid a reference on c/image/storage.
+		// See https://github.com/containers/skopeo/issues/771#issuecomment-563125006.
+		if transport != nil && transport.Name() == "containers-storage" {
+			return maybeReexec()
+		}
+	}
+	return nil
+}