Add Prefix config for S3 storage (#1812)

* feat: add Prefix config for S3 storage * feat: remove values * feat: add s3_prefix column to configurations table * feat: copy s3_prefix field in transform * feat: edit test for all config fields * fix: fix tests
frain-dev · Oct 26, 2023 · d55b74e · d55b74e
1 parent 2302037
commit d55b74e
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 12 deletions.
diff --git a/api/models/configuration.go b/api/models/configuration.go
@@ -40,6 +40,7 @@ func (sc *StoragePolicyConfiguration) Transform() *datastore.StoragePolicyConfig
 }
 
 type S3Storage struct {
+	Prefix       null.String `json:"prefix"`
 	Bucket       null.String `json:"bucket" valid:"required~please provide a bucket name"`
 	AccessKey    null.String `json:"access_key,omitempty" valid:"required~please provide an access key"`
 	SecretKey    null.String `json:"secret_key,omitempty" valid:"required~please provide a secret key"`
@@ -54,6 +55,7 @@ func (s3 *S3Storage) transform() *datastore.S3Storage {
 	}
 
 	return &datastore.S3Storage{
+		Prefix:       s3.Prefix,
 		Bucket:       s3.Bucket,
 		AccessKey:    s3.AccessKey,
 		SecretKey:    s3.SecretKey,

diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go
@@ -174,6 +174,7 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat
 	configRepo := postgres.NewConfigRepo(a.DB)
 
 	s3 := datastore.S3Storage{
+		Prefix:       null.NewString(cfg.StoragePolicy.S3.Prefix, true),
 		Bucket:       null.NewString(cfg.StoragePolicy.S3.Bucket, true),
 		AccessKey:    null.NewString(cfg.StoragePolicy.S3.AccessKey, true),
 		SecretKey:    null.NewString(cfg.StoragePolicy.S3.SecretKey, true),

diff --git a/config/config.go b/config/config.go
@@ -240,6 +240,7 @@ type StoragePolicyConfiguration struct {
 }
 
 type S3Storage struct {
+	Prefix       string `json:"prefix" envconfig:"CONVOY_STORAGE_AWS_PREFIX"`
 	Bucket       string `json:"bucket" envconfig:"CONVOY_STORAGE_AWS_BUCKET"`
 	AccessKey    string `json:"access_key" envconfig:"CONVOY_STORAGE_AWS_ACCESS_KEY"`
 	SecretKey    string `json:"secret_key" envconfig:"CONVOY_STORAGE_AWS_SECRET_KEY"`

diff --git a/database/postgres/configuration.go b/database/postgres/configuration.go
@@ -14,16 +14,16 @@ import (
 const (
 	createConfiguration = `
 	INSERT INTO convoy.configurations(
-		id, is_analytics_enabled, is_signup_enabled, 
-		storage_policy_type, on_prem_path, 
-		s3_bucket, s3_access_key, s3_secret_key, 
+		id, is_analytics_enabled, is_signup_enabled,
+		storage_policy_type, on_prem_path, s3_prefix,
+		s3_bucket, s3_access_key, s3_secret_key,
 		s3_region, s3_session_token, s3_endpoint
-	  ) 
-	  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);
+	  )
+	  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12);
 	`
 
 	fetchConfiguration = `
-	SELECT 
+	SELECT
 		id,
 		is_analytics_enabled,
 		is_signup_enabled,
@@ -35,6 +35,7 @@ const (
 		s3_region AS "storage_policy.s3.region",
 		s3_session_token AS "storage_policy.s3.session_token",
 		s3_endpoint AS "storage_policy.s3.endpoint",
+		s3_prefix AS "storage_policy.s3.prefix",
 		created_at,
 		updated_at,
 		deleted_at
@@ -54,8 +55,9 @@ const (
 		s3_access_key = $7,
 		s3_secret_key = $8,
 		s3_region = $9,
-		s3_session_token = $10, 
+		s3_session_token = $10,
 		s3_endpoint = $11,
+		s3_prefix = $12,
 		updated_at = now()
 	WHERE id = $1 AND deleted_at IS NULL;
 	`
@@ -72,6 +74,7 @@ func NewConfigRepo(db database.Database) datastore.ConfigurationRepository {
 func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore.Configuration) error {
 	if config.StoragePolicy.Type == datastore.OnPrem {
 		config.StoragePolicy.S3 = &datastore.S3Storage{
+			Prefix:       null.NewString("", false),
 			Bucket:       null.NewString("", false),
 			AccessKey:    null.NewString("", false),
 			SecretKey:    null.NewString("", false),
@@ -91,6 +94,7 @@ func (c *configRepo) CreateConfiguration(ctx context.Context, config *datastore.
 		config.IsSignupEnabled,
 		config.StoragePolicy.Type,
 		config.StoragePolicy.OnPrem.Path,
+		config.StoragePolicy.S3.Prefix,
 		config.StoragePolicy.S3.Bucket,
 		config.StoragePolicy.S3.AccessKey,
 		config.StoragePolicy.S3.SecretKey,
@@ -130,6 +134,7 @@ func (c *configRepo) LoadConfiguration(ctx context.Context) (*datastore.Configur
 func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Configuration) error {
 	if cfg.StoragePolicy.Type == datastore.OnPrem {
 		cfg.StoragePolicy.S3 = &datastore.S3Storage{
+			Prefix:       null.NewString("", false),
 			Bucket:       null.NewString("", false),
 			AccessKey:    null.NewString("", false),
 			SecretKey:    null.NewString("", false),
@@ -155,8 +160,8 @@ func (c *configRepo) UpdateConfiguration(ctx context.Context, cfg *datastore.Con
 		cfg.StoragePolicy.S3.Region,
 		cfg.StoragePolicy.S3.SessionToken,
 		cfg.StoragePolicy.S3.Endpoint,
+		cfg.StoragePolicy.S3.Prefix,
 	)
-
 	if err != nil {
 		return err
 	}

diff --git a/database/postgres/configuration_test.go b/database/postgres/configuration_test.go
@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/guregu/null.v4"
+
 	"github.com/oklog/ulid/v2"
 
 	"github.com/frain-dev/convoy/datastore"
@@ -93,6 +95,20 @@ func generateConfig() *datastore.Configuration {
 		UID:                ulid.Make().String(),
 		IsAnalyticsEnabled: true,
 		IsSignupEnabled:    false,
-		StoragePolicy:      &datastore.DefaultStoragePolicy,
+		StoragePolicy: &datastore.StoragePolicyConfiguration{
+			Type: datastore.OnPrem,
+			S3: &datastore.S3Storage{
+				Prefix:       null.NewString("random7", true),
+				Bucket:       null.NewString("random1", true),
+				AccessKey:    null.NewString("random2", true),
+				SecretKey:    null.NewString("random3", true),
+				Region:       null.NewString("random4", true),
+				SessionToken: null.NewString("random5", true),
+				Endpoint:     null.NewString("random6", true),
+			},
+			OnPrem: &datastore.OnPremStorage{
+				Path: null.NewString("path", true),
+			},
+		},
 	}
 }
diff --git a/datastore/models.go b/datastore/models.go
@@ -1221,6 +1221,7 @@ type StoragePolicyConfiguration struct {
 }
 
 type S3Storage struct {
+	Prefix       null.String `json:"prefix" db:"prefix"`
 	Bucket       null.String `json:"bucket" db:"bucket" valid:"required~please provide a bucket name"`
 	AccessKey    null.String `json:"access_key,omitempty" db:"access_key" valid:"required~please provide an access key"`
 	SecretKey    null.String `json:"secret_key,omitempty" db:"secret_key" valid:"required~please provide a secret key"`

diff --git a/datastore/object-store/objectstore.go b/datastore/object-store/objectstore.go
@@ -5,6 +5,7 @@ type ObjectStore interface {
 }
 
 type ObjectStoreOptions struct {
+	Prefix           string
 	Bucket           string
 	AccessKey        string
 	SecretKey        string

diff --git a/datastore/object-store/s3.go b/datastore/object-store/s3.go
@@ -4,6 +4,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/frain-dev/convoy/util"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -32,7 +34,6 @@ func NewS3Client(opts ObjectStoreOptions) (ObjectStore, error) {
 	}
 
 	return client, nil
-
 }
 
 func (s3 *S3Client) Save(filename string) error {
@@ -44,11 +45,21 @@ func (s3 *S3Client) Save(filename string) error {
 
 	defer file.Close()
 
-	names := strings.Split(filename, "/tmp/")
+	name := filename
+
+	if util.IsStringEmpty(s3.opts.Prefix) {
+		names := strings.Split(filename, "/tmp/")
+		if len(names) > 1 {
+			name = names[1]
+		}
+	} else {
+		name = strings.Replace(filename, "/tmp", s3.opts.Prefix, 1)
+	}
+
 	uploader := s3manager.NewUploader(s3.session)
 	_, err = uploader.Upload(&s3manager.UploadInput{
 		Bucket: aws.String(s3.opts.Bucket),
-		Key:    aws.String(names[1]),
+		Key:    aws.String(name),
 		Body:   file,
 	})
 

diff --git a/sql/1698074481.sql b/sql/1698074481.sql
@@ -0,0 +1,6 @@
+-- +migrate Up
+ALTER TABLE convoy.configurations ADD COLUMN s3_prefix text;
+
+-- +migrate Down
+ALTER TABLE convoy.configurations DROP COLUMN s3_prefix;
+
diff --git a/worker/task/retention_policies.go b/worker/task/retention_policies.go
@@ -74,13 +74,15 @@ func NewObjectStoreClient(storage *datastore.StoragePolicyConfiguration) (object
 	case datastore.S3:
 		exportDir := convoy.TmpExportDir
 		objectStoreOpts := objectstore.ObjectStoreOptions{
+			Prefix:       storage.S3.Prefix.ValueOrZero(),
 			Bucket:       storage.S3.Bucket.ValueOrZero(),
 			Endpoint:     storage.S3.Endpoint.ValueOrZero(),
 			AccessKey:    storage.S3.AccessKey.ValueOrZero(),
 			SecretKey:    storage.S3.SecretKey.ValueOrZero(),
 			SessionToken: storage.S3.SessionToken.ValueOrZero(),
 			Region:       storage.S3.Region.ValueOrZero(),
 		}
+
 		objectStoreClient, err := objectstore.NewS3Client(objectStoreOpts)
 		if err != nil {
 			return nil, "", err