Skip to content

Commit

Permalink
Support for sub-folders when mounting S3 buckets (#263)
Browse files Browse the repository at this point in the history 
* initial support for s3 folders

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* adding support for building local component containers with podman

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* bug fixes in build bash scripts

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* bug fixes in build bash scripts

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* bug fixes in build bash scripts

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* bug fixes in build bash scripts

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* bug fixes in build bash scripts

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* bug fixes in build bash scripts

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* docker buildx load after build

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

* docker buildx load after build

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>

---------

Signed-off-by: SRIKUMAR VENUGOPAL <srikumarv@ie.ibm.com>
  • Loading branch information
@srikumar003
srikumar003 authored Jan 22, 2024
1 parent b4c16e4 commit 24062e9
Show file tree
Hide file tree
Showing 14 changed files with 301 additions and 67 deletions.
45 changes: 38 additions & 7 deletions build-tools/build_components.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,47 @@ do
esac
done

shift $((OPTIND-1))

REGISTRY_URL="${1:-quay.io/datashim-io}"
VERSION="${2:-latest}"

DOCKERCMD="docker"
ALTDOCKERCMD="podman"
if !(command -v ${DOCKERCMD} &> /dev/null)
then
echo "Docker command not found"
if !(command -v ${ALTDOCKERCMD} &> /dev/null)
then
echo "Neither ${DOCKERCMD} nor ${ALTDOCKERCMD} commands found.. cannot build "
exit 1
else
DOCKERCMD=${ALTDOCKERCMD}
fi
else
echo "Docker command found"
cmd_type=$(type -t ${DOCKERCMD})
if [ $cmd_type == "alias" ]
then
echo "${DOCKERCMD} is an alias, switching to ${ALTDOCKERCMD}"
DOCKERCMD=${ALTDOCKERCMD}
fi
fi

if [ ${DOCKERCMD} == "docker" ]
then
if [ $CREATE_NEW_BUILDX_CONTEXT = "yes" ]; then
docker buildx create --use
fi
fi

if [ $BUILD_AND_PUSH = "yes" ]; then
if [ $SKIP_LOGIN = "no" ]; then
echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USERNAME --password-stdin $REGISTRY_URL
fi
if [ $CREATE_NEW_BUILDX_CONTEXT = "yes" ]; then
docker buildx create --use
fi
(cd ../src/dataset-operator && ./build_and_push_multiarch_dataset_operator.sh $REGISTRY_URL)
(cd ../src/generate-keys && ./build_and_push_multiarch_generate_keys.sh $REGISTRY_URL)
(cd ../src/dataset-operator && ./build_multiarch_dataset_operator.sh -p $REGISTRY_URL $VERSION)
(cd ../src/generate-keys && ./build_multiarch_generate_keys.sh -p $REGISTRY_URL $VERSION)
else
(cd ../src/dataset-operator && ./build_dataset_operator.sh $REGISTRY_URL)
(cd ../src/generate-keys && ./build_generate_keys.sh $REGISTRY_URL)
(cd ../src/dataset-operator && ./build_multiarch_dataset_operator.sh $REGISTRY_URL $VERSION)
(cd ../src/generate-keys && ./build_multiarch_generate_keys.sh $REGISTRY_URL $VERSION)
fi
1 change: 1 addition & 0 deletions src/csi-s3/pkg/s3/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ type Config struct {
Endpoint string
Mounter string
Bucket string
Folder string
RemoveOnDelete bool
Readonly bool
Provision bool
Expand Down
49 changes: 28 additions & 21 deletions src/csi-s3/pkg/s3/mounter_goofys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package s3

import (
"fmt"
"github.com/golang/glog"
"log"
"os"

"github.com/golang/glog"
)

const (
goofysCmd = "goofys"
goofysCmd = "goofys"
)

// Implements Mounter
Expand All @@ -18,12 +19,12 @@ type goofysMounter struct {
region string
accessKeyID string
secretAccessKey string
volumeID string
readonly bool
volumeID string
readonly bool
}

func newGoofysMounter(b *bucket, cfg *Config, volume string) (Mounter, error) {
glog.V(3).Infof("Mounting using goofys volume %s",volume)
glog.V(3).Infof("Mounting using goofys volume %s", volume)
//TODO we need to handle regions as well
//region := cfg.Region
//// if endpoint is set we need a default region
Expand All @@ -36,8 +37,8 @@ func newGoofysMounter(b *bucket, cfg *Config, volume string) (Mounter, error) {
region: cfg.Region,
accessKeyID: cfg.AccessKeyID,
secretAccessKey: cfg.SecretAccessKey,
readonly: cfg.Readonly,
volumeID: volume,
readonly: cfg.Readonly,
volumeID: volume,
}, nil
}

Expand All @@ -57,11 +58,11 @@ func (goofys *goofysMounter) Mount(source string, target string) error {
}
args := []string{
fmt.Sprintf("--endpoint=%s", goofys.endpoint),
"--type-cache-ttl","1s",
"--stat-cache-ttl","1s",
"--dir-mode","0777",
"--file-mode","0777",
"--http-timeout","5m",
"--type-cache-ttl", "1s",
"--stat-cache-ttl", "1s",
"--dir-mode", "0777",
"--file-mode", "0777",
"--http-timeout", "5m",
//fmt.Sprintf("--cheap=%s", os.Getenv("cheap")),
"-o", "allow_other",
}
Expand All @@ -71,21 +72,28 @@ func (goofys *goofysMounter) Mount(source string, target string) error {
if goofys.region != "" {
args = append(args, "--region", goofys.region)
}
if(goofys.readonly) {
args = append(args, "-o","ro")
if goofys.readonly {
args = append(args, "-o", "ro")
}
if goofys.bucket.Folder == "" {
glog.V(4).Infof("This bucket %s contains no folder prefixes", goofys.bucket.Name)
args = append(args,
fmt.Sprintf("%s", goofys.bucket.Name),
fmt.Sprintf("%s", target))
} else {
glog.V(4).Infof("This bucket %s contains folder prefix %s", goofys.bucket.Name, goofys.bucket.Folder)
args = append(args,
fmt.Sprintf("%s:%s", goofys.bucket.Name, goofys.bucket.Folder),
fmt.Sprintf("%s", target))
}
args = append(args,
fmt.Sprintf("%s", goofys.bucket.Name),
fmt.Sprintf("%s", target))
return fuseMount(target, goofysCmd, args)

}


func writes3fsPassGoofy(goofys *goofysMounter) error {
awsPath := fmt.Sprintf("%s/.aws", os.Getenv("HOME"))
if _, err := os.Stat(awsPath); os.IsNotExist(err) {
mkdir_err := os.Mkdir(awsPath,0700)
mkdir_err := os.Mkdir(awsPath, 0700)
if mkdir_err != nil {
return mkdir_err
}
Expand All @@ -98,10 +106,9 @@ func writes3fsPassGoofy(goofys *goofysMounter) error {
log.Println(err)
}
defer f.Close()
textToAdd := "["+goofys.volumeID+"]\naws_access_key_id = "+goofys.accessKeyID+"\naws_secret_access_key ="+goofys.secretAccessKey+"\n"
textToAdd := "[" + goofys.volumeID + "]\naws_access_key_id = " + goofys.accessKeyID + "\naws_secret_access_key =" + goofys.secretAccessKey + "\n"
if _, err := f.WriteString(textToAdd); err != nil {
log.Println(err)
}
return nil
}

10 changes: 10 additions & 0 deletions src/csi-s3/pkg/s3/nodeserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,15 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
return nil, status.Error(codes.InvalidArgument, "Bucket name not provided for mounting")
}

// srikumarv - this is a hack to support folders for the current csi-s3 implementation used in
// Datashim
folder := ""
if len(s3.cfg.Folder) != 0 {
folder = s3.cfg.Folder
} else {
glog.V(2).Infof("s3: no fspath found for bucket %s", bucketName)
}

//b, err := s3.getBucket(bucketName)
//if err != nil {
// return nil, err
Expand All @@ -130,6 +139,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis

b := &bucket{
Name: bucketName,
Folder: folder,
Mounter: volContext[mounterTypeKey],
}

Expand Down
4 changes: 3 additions & 1 deletion src/csi-s3/pkg/s3/s3-client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type s3Client struct {

type bucket struct {
Name string
Folder string
Mounter string
}

Expand Down Expand Up @@ -54,6 +55,7 @@ func newS3ClientFromSecrets(secrets map[string]string) (*s3Client, error) {
Region: secrets["region"],
Endpoint: secrets["endpoint"],
Bucket: secrets["bucket"],
Folder: secrets["folder"],
Readonly: readonly,
Provision: provision,
RemoveOnDelete: removeOnDelete,
Expand Down Expand Up @@ -127,7 +129,7 @@ func (client *s3Client) emptyBucket(bucketName string) error {
// return err
//}

//TODO is it not used at all?
// TODO is it not used at all?
func (client *s3Client) metadataExist(bucketName string) bool {
opts := minio.GetObjectOptions{}
_, err := client.minio.GetObject(bucketName, metadataName, opts)
Expand Down
5 changes: 0 additions & 5 deletions src/dataset-operator/build_and_push_multiarch_dataset_operator.sh
View file

This file was deleted.

5 changes: 0 additions & 5 deletions src/dataset-operator/build_dataset_operator.sh
View file

This file was deleted.

86 changes: 86 additions & 0 deletions src/dataset-operator/build_multiarch_dataset_operator.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#!/bin/bash
set -e

print_usage() {
echo "usage: $0 [-p] <REGISTRY_URL> <VERSION>"
echo "Use -p to build and push multiarch images"
}

PUSH="false"
while getopts 'p' flag; do
case "$flag" in
p)
PUSH="true"
;;
?)
print_usage >&2
exit 1
;;
esac
done

shift $((OPTIND-1))

REGISTRY_URL="${1:-quay.io/datashim-io}"
VERSION="${2:-latest}"

docker_build () {
docker buildx build --platform linux/amd64 -t ${REGISTRY_URL}/dataset-operator:${VERSION} .
docker buildx build --load -t ${REGISTRY_URL}/dataset-operator:${VERSION} .
}

docker_build_and_push () {
docker buildx build --platform linux/amd64,linux/arm64,linux/ppc64le --push -t ${REGISTRY_URL}/dataset-operator:${VERSION} .
}

podman_build () {
podman manifest create ${REGISTRY_URL}/dataset-operator:${VERSION}
podman buildx build --platform linux/amd64,linux/arm64 --manifest ${REGISTRY_URL}/dataset-operator:${VERSION} .
}

podman_push () {
podman manifest push ${REGISTRY_URL}/dataset-operator:${VERSION}

}

DOCKERCMD="docker"
ALTDOCKERCMD="podman"
if !(command -v ${DOCKERCMD} &> /dev/null)
then
echo "Docker command not found"
if !(command -v ${ALTDOCKERCMD} &> /dev/null)
then
echo "Neither ${DOCKERCMD} nor ${ALTDOCKERCMD} commands found.. cannot build "
exit 1
else
DOCKERCMD=${ALTDOCKERCMD}
fi
else
echo "Docker command found"
cmd_type=$(type -t ${DOCKERCMD})
if [ $cmd_type == "alias" ]
then
echo "${DOCKERCMD} is an alias, switching to ${ALTDOCKERCMD}"
DOCKERCMD=${ALTDOCKERCMD}
fi
fi

if [ $PUSH == "true" ]
then
echo "pushing images to the registry"
if [ ${DOCKERCMD} == "docker" ]
then
docker_build_and_push
else
podman_build
podman_push
fi
else
echo "building image locally"
if [ ${DOCKERCMD} == "docker" ]
then
docker_build
else
podman_build
fi
fi
Loading

0 comments on commit 24062e9

Please sign in to comment.