Swith s3 ListObjects api to ListObjectsV2 (#2006)

Switch the ListObjects API used in s3 storage to ListObjectsV2
gomods · Nov 26, 2024 · 8f0ee5e · 8f0ee5e
1 parent bf38a47
commit 8f0ee5e
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 7 deletions.
diff --git a/pkg/storage/s3/cataloger.go b/pkg/storage/s3/cataloger.go
@@ -24,12 +24,12 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
 	res := make([]paths.AllPathParams, 0)
 	count := pageSize
 	for count > 0 {
-		lsParams := &s3.ListObjectsInput{
-			Bucket: aws.String(s.bucket),
-			Marker: &queryToken,
+		lsParams := &s3.ListObjectsV2Input{
+			Bucket:     aws.String(s.bucket),
+			StartAfter: &queryToken,
 		}
 
-		loo, err := s.s3API.ListObjects(ctx, lsParams)
+		loo, err := s.s3API.ListObjectsV2(ctx, lsParams)
 		if err != nil {
 			return nil, "", errors.E(op, err)
 		}

diff --git a/pkg/storage/s3/lister.go b/pkg/storage/s3/lister.go
@@ -19,12 +19,13 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
 	defer span.End()
 
 	modulePrefix := strings.TrimSuffix(module, "/") + "/@v"
-	lsParams := &s3.ListObjectsInput{
+
+	lsParams := &s3.ListObjectsV2Input{
 		Bucket: aws.String(s.bucket),
 		Prefix: aws.String(modulePrefix),
 	}
 
-	loo, err := s.s3API.ListObjects(ctx, lsParams)
+	loo, err := s.s3API.ListObjectsV2(ctx, lsParams)
 	if err != nil {
 		return nil, errors.E(op, err, errors.M(module))
 	}

diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go
@@ -30,7 +30,7 @@ func (s *Storage) clear() error {
 	ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
 	defer cancel()
 
-	objects, err := s.s3API.ListObjects(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)})
+	objects, err := s.s3API.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket)})
 	if err != nil {
 		return err
 	}