streamline archive handling

bindings-go/ctf/comp_test.go

@@ -0,0 +1,207 @@
+// Copyright 2020 Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ctf_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/gardener/component-spec/bindings-go/ctf"
+	"github.com/mandelsoft/vfs/pkg/layerfs"
+	"github.com/mandelsoft/vfs/pkg/osfs"
+	"github.com/mandelsoft/vfs/pkg/projectionfs"
+	"github.com/mandelsoft/vfs/pkg/vfs"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	v2 "github.com/gardener/component-spec/bindings-go/apis/v2"
+	"github.com/mandelsoft/vfs/pkg/memoryfs"
+	"github.com/opencontainers/go-digest"
+)
+
+func OpenTestComponent(path string) (*ctf.ComponentArchive, error) {
+	fs, err := projectionfs.New(osfs.New(), path)
+	if err != nil {
+		return nil, err
+	}
+	layer := memoryfs.New() // osfs.NewTempFileSystem()
+	fs = layerfs.New(layer, fs)
+	return ctf.NewComponentArchiveFromFilesystem(fs)
+}
+
+var _ = Describe("ComponentArchive", func() {
+
+	Context("build", func() {
+		It("should build a component archive from path", func() {
+			ctx := context.Background()
+			defer ctx.Done()
+			ca, err := OpenTestComponent("./testdata/component-01")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ca.ComponentDescriptor.Resources).To(HaveLen(1))
+
+			var data bytes.Buffer
+			info, err := ca.Resolve(ctx, ca.ComponentDescriptor.Resources[0], &data)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(info.MediaType).To(Equal("json"))
+			Expect(data.Bytes()).To(Equal([]byte("{\"some\": \"data\"}")))
+		})
+
+		It("should build a component archive from a tar", func() {
+			ctx := context.Background()
+			defer ctx.Done()
+			ca, err := OpenTestComponent("./testdata/component-01")
+			Expect(err).ToNot(HaveOccurred())
+
+			file, err := ioutil.TempFile(os.TempDir(), "ca-tar-")
+			Expect(err).ToNot(HaveOccurred())
+			defer func() {
+				Expect(os.Remove(file.Name())).ToNot(HaveOccurred())
+			}()
+			Expect(ca.WriteTar(file)).To(Succeed())
+
+			ca, err = ctf.ComponentArchiveFromCTF(file.Name())
+			Expect(err).ToNot(HaveOccurred())
+
+			var data bytes.Buffer
+			info, err := ca.Resolve(ctx, ca.ComponentDescriptor.Resources[0], &data)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(info.MediaType).To(Equal("json"))
+			Expect(data.Bytes()).To(Equal([]byte("{\"some\": \"data\"}")))
+		})
+	})
+
+	It("should build a tar from a component archive", func() {
+		ctx := context.Background()
+		defer ctx.Done()
+		ca, err := OpenTestComponent("./testdata/component-01")
+		Expect(err).ToNot(HaveOccurred())
+
+		var data bytes.Buffer
+		Expect(ca.WriteTar(&data)).To(Succeed())
+
+		fs := memoryfs.New()
+		Expect(ctf.ExtractTarToFs(fs, &data)).To(Succeed())
+
+		_, err = fs.Stat(ctf.ComponentDescriptorFileName)
+		Expect(err).ToNot(HaveOccurred())
+		blobData, err := vfs.ReadFile(fs, "blobs/myblob")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(blobData).To(Equal([]byte("{\"some\": \"data\"}")))
+	})
+
+	It("should write a component archive to a filesystem", func() {
+		ctx := context.Background()
+		defer ctx.Done()
+		ca, err := OpenTestComponent("./testdata/component-01")
+		Expect(err).ToNot(HaveOccurred())
+
+		fs := memoryfs.New()
+		Expect(ca.WriteToFilesystem(fs, "test")).To(Succeed())
+
+		_, err = fs.Stat(vfs.Join(fs, "test", ctf.ComponentDescriptorFileName))
+		Expect(err).ToNot(HaveOccurred())
+		blobData, err := vfs.ReadFile(fs, "test/blobs/myblob")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(blobData).To(Equal([]byte("{\"some\": \"data\"}")))
+	})
+
+	It("should add a resource to the component archive from a data reader", func() {
+		ctx := context.Background()
+		defer ctx.Done()
+		ca, err := OpenTestComponent("./testdata/component-01")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(ca.ComponentDescriptor.Resources).To(HaveLen(1))
+
+		data := []byte("test")
+		info := &ctf.BlobInfo{
+			MediaType: "txt",
+			Digest:    digest.FromBytes(data).String(),
+			Size:      int64(len(data)),
+		}
+		res := v2.Resource{
+			IdentityObjectMeta: v2.IdentityObjectMeta{
+				Name:    "res1",
+				Type:    "txt",
+				Version: "v1.1",
+			},
+			Relation: v2.ExternalRelation,
+		}
+		Expect(ca.AddResource(&res, *info, bytes.NewBuffer(data)))
+
+		Expect(ca.ComponentDescriptor.Resources).To(HaveLen(2))
+		var result bytes.Buffer
+		info, err = ca.Resolve(ctx, res, &result)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(info.MediaType).To(Equal("txt"))
+		Expect(result.Bytes()).To(Equal(data))
+	})
+
+	It("should add a resource to the component archive from a blobresolver", func() {
+		ctx := context.Background()
+		defer ctx.Done()
+		ca, err := OpenTestComponent("./testdata/component-01")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(ca.ComponentDescriptor.Resources).To(HaveLen(1))
+
+		data := []byte("test")
+		info := &ctf.BlobInfo{
+			MediaType: "txt",
+			Digest:    digest.FromBytes(data).String(),
+			Size:      int64(len(data)),
+		}
+		res := v2.Resource{
+			IdentityObjectMeta: v2.IdentityObjectMeta{
+				Name:    "res1",
+				Type:    "txt",
+				Version: "v1.1",
+			},
+
+			Relation: v2.ExternalRelation,
+		}
+		blobresolver := &testBlobResolver{
+			info: func(ctx context.Context, res v2.Resource) (*ctf.BlobInfo, error) { return info, nil },
+			resolve: func(ctx context.Context, res v2.Resource, writer io.Writer) (*ctf.BlobInfo, error) {
+				if _, err := io.Copy(writer, bytes.NewBuffer(data)); err != nil {
+					return nil, err
+				}
+				return info, nil
+			},
+		}
+		Expect(ca.AddResourceFromResolver(ctx, &res, blobresolver))
+
+		Expect(ca.ComponentDescriptor.Resources).To(HaveLen(2))
+		var result bytes.Buffer
+		info, err = ca.Resolve(ctx, res, &result)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(info.MediaType).To(Equal("txt"))
+		Expect(result.Bytes()).To(Equal(data))
+	})
+
+})
+
+type testBlobResolver struct {
+	info    func(ctx context.Context, res v2.Resource) (*ctf.BlobInfo, error)
+	resolve func(ctx context.Context, res v2.Resource, writer io.Writer) (*ctf.BlobInfo, error)
+}
+
+func (t *testBlobResolver) Info(ctx context.Context, res v2.Resource) (*ctf.BlobInfo, error) {
+	return t.info(ctx, res)
+}
+func (t *testBlobResolver) Resolve(ctx context.Context, res v2.Resource, writer io.Writer) (*ctf.BlobInfo, error) {
+	return t.resolve(ctx, res, writer)
+}

bindings-go/ctf/componentarchive.go

@@ -47,6 +47,46 @@ func NewComponentArchive(cd *v2.ComponentDescriptor, fs vfs.FileSystem) *Compone
 	}
 }
 
+func NewComponentArchiveFromPath(fs vfs.FileSystem, path string) (*ComponentArchive, error) {
+	return OpenComponentArchive(fs, path)
+}
+
+func OpenComponentArchive(fs vfs.FileSystem, path string) (*ComponentArchive, error) {
+	if fs == nil {
+		fs = osfs.New()
+	}
+	fi, err := fs.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, vfs.ErrNotExist
+		}
+		return nil, err
+	}
+	if fi.IsDir() {
+		cfs, err := projectionfs.New(fs, path)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create projected filesystem from path %s: %w", path, err)
+		}
+		return NewComponentArchiveFromFilesystem(cfs)
+	}
+
+	file, err := fs.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open tar archive from %s: %w", path, err)
+	}
+
+	reader, err := gzip.NewReader(file)
+	if err == nil {
+		defer file.Close()
+		return NewComponentArchiveFromTarReader(reader)
+	}
+	file.Close()
+	file, err = fs.Open(path)
+
+	defer file.Close()
+	return NewComponentArchiveFromTarReader(file)
+}
+
 // ComponentArchiveFromPath creates a component archive from a path
 func ComponentArchiveFromPath(path string) (*ComponentArchive, error) {
 	fs, err := projectionfs.New(osfs.New(), path)
@@ -98,7 +138,7 @@ func NewComponentArchiveFromTarReader(in io.Reader) (*ComponentArchive, error) {
 
 // NewComponentArchiveFromFilesystem creates a new component archive from a filesystem.
 func NewComponentArchiveFromFilesystem(fs vfs.FileSystem, decodeOpts ...codec.DecodeOption) (*ComponentArchive, error) {
-	data, err := vfs.ReadFile(fs, filepath.Join("/", ComponentDescriptorFileName))
+	data, err := vfs.ReadFile(fs, vfs.Join(fs, "/", ComponentDescriptorFileName))
 	if err != nil {
 		return nil, fmt.Errorf("unable to read the component descriptor from %s: %w", ComponentDescriptorFileName, err)
 	}
@@ -133,6 +173,17 @@ func (ca *ComponentArchive) Digest() (string, error) {
 	return digest.FromBytes(data).String(), nil
 }
 
+func (ca *ComponentArchive) writeCD() error {
+	cdBytes, err := codec.Encode(ca.ComponentDescriptor)
+	if err != nil {
+		return fmt.Errorf("unable to encode component descriptor: %w", err)
+	}
+	if err := vfs.WriteFile(ca.fs, ComponentDescriptorFileName, cdBytes, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to copy component descritptor to %q: %w", ComponentDescriptorFileName, err)
+	}
+	return nil
+}
+
 // AddResource adds a blob resource to the current archive.
 // If the specified resource already exists it will be overwritten.
 func (ca *ComponentArchive) AddResource(res *v2.Resource, info BlobInfo, reader io.Reader) error {
@@ -173,7 +224,7 @@ func (ca *ComponentArchive) AddResource(res *v2.Resource, info BlobInfo, reader
 	} else {
 		ca.ComponentDescriptor.Resources[id] = *res
 	}
-	return nil
+	return ca.writeCD()
 }
 
 // AddSource adds a blob source to the current archive.
@@ -216,7 +267,7 @@ func (ca *ComponentArchive) AddSource(src *v2.Source, info BlobInfo, reader io.R
 	} else {
 		ca.ComponentDescriptor.Sources[id] = *src
 	}
-	return nil
+	return ca.writeCD()
 }
 
 // AddResourceFromResolver adds a blob resource to the current archive.
@@ -264,7 +315,7 @@ func (ca *ComponentArchive) AddResourceFromResolver(ctx context.Context, res *v2
 	} else {
 		ca.ComponentDescriptor.Resources[id] = *res
 	}
-	return nil
+	return ca.writeCD()
 }
 
 // ensureBlobsPath ensures that the blob directory exists
@@ -278,6 +329,17 @@ func (ca *ComponentArchive) ensureBlobsPath() error {
 	return nil
 }
 
+// WriteToArchive writes an archive of the given type.
+func (ca *ComponentArchive) WriteToArchive(writer io.Writer, format ArchiveFormat) error {
+	if format == ArchiveFormatTar {
+		return ca.WriteTar(writer)
+	}
+	if format == ArchiveFormatTarGzip {
+		return ca.WriteTarGzip(writer)
+	}
+	return fmt.Errorf("invalid archive format '%s'", format)
+}
+
 // WriteTarGzip tars the current components descriptor and its artifacts.
 func (ca *ComponentArchive) WriteTarGzip(writer io.Writer) error {
 	gw := gzip.NewWriter(writer)
@@ -358,16 +420,16 @@ func (ca *ComponentArchive) WriteTar(writer io.Writer) error {
 // WriteToFilesystem writes the current component archive to a filesystem
 func (ca *ComponentArchive) WriteToFilesystem(fs vfs.FileSystem, path string) error {
 	// create the directory structure with the blob directory
-	if err := fs.MkdirAll(filepath.Join(path, BlobsDirectoryName), os.ModePerm); err != nil {
+	if err := fs.MkdirAll(vfs.Join(fs, path, BlobsDirectoryName), os.ModePerm); err != nil {
 		return fmt.Errorf("unable to create output directory %q: %s", path, err.Error())
 	}
 	// copy component-descriptor
 	cdBytes, err := codec.Encode(ca.ComponentDescriptor)
 	if err != nil {
 		return fmt.Errorf("unable to encode component descriptor: %w", err)
 	}
-	if err := vfs.WriteFile(fs, filepath.Join(path, ComponentDescriptorFileName), cdBytes, os.ModePerm); err != nil {
-		return fmt.Errorf("unable to copy component descritptor to %q: %w", filepath.Join(path, ComponentDescriptorFileName), err)
+	if err := vfs.WriteFile(fs, vfs.Join(fs, path, ComponentDescriptorFileName), cdBytes, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to copy component descritptor to %q: %w", vfs.Join(fs, path, ComponentDescriptorFileName), err)
 	}
 
 	// copy all blobs
@@ -383,7 +445,7 @@ func (ca *ComponentArchive) WriteToFilesystem(fs vfs.FileSystem, path string) er
 			continue
 		}
 		inpath := BlobPath(blobInfo.Name())
-		outpath := filepath.Join(path, BlobsDirectoryName, blobInfo.Name())
+		outpath := vfs.Join(fs, path, BlobsDirectoryName, blobInfo.Name())
 		blob, err := ca.fs.Open(inpath)
 		if err != nil {
 			return fmt.Errorf("unable to open input blob %q: %w", inpath, err)