Swtich to list from walk as it seems more stble for S3.

gorpipe · Dec 25, 2024 · c066fb5 · c066fb5
1 parent 386d078
commit c066fb5
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java b/drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java
@@ -135,8 +135,6 @@ private S3Client createClient(Credentials cred) {
 
         builder.httpClientBuilder(httpClientBuilder);
 
-
-
         var metricsPub = new PrometheusMetricPublisher();
         builder.overrideConfiguration(c -> c.addMetricPublisher(metricsPub));
 
@@ -157,8 +155,8 @@ private S3Client createClient(Credentials cred) {
     AwsCredentialsProvider getCredentialsProvider(Credentials cred) {
         if (cred == null || cred.isNull()) {
             log.info("CredentialsProvider: DefaultCredentialsProvider for null creds");
-            throw new IllegalArgumentException("Credentials are required for S3 access");
-            //return DefaultCredentialsProvider.create();
+            //throw new IllegalArgumentException("Credentials are required for S3 access");
+            return DefaultCredentialsProvider.create();
         } else {
             var awsKey = cred.getOrDefault(Credentials.Attr.KEY, "");
             var awsSecret = cred.getOrDefault(Credentials.Attr.SECRET, "");
@@ -182,8 +180,8 @@ AwsCredentialsProvider getCredentialsProvider(Credentials cred) {
                 }
             } else {
                 log.info(String.format("CredentialsProvider: DefaultCredentialsProvider for %s:%s", cred.getService(), cred.getLookupKey()));
-                throw new IllegalArgumentException(String.format("CredentialsProvider: DefaultCredentialsProvider for %s:%s", cred.getService(), cred.getLookupKey()));
-                //return DefaultCredentialsProvider.create();
+                //throw new IllegalArgumentException(String.format("CredentialsProvider: DefaultCredentialsProvider for %s:%s", cred.getService(), cred.getLookupKey()));
+                return DefaultCredentialsProvider.create();
             }
         }
     }

diff --git a/drivers/src/test/java/org/gorpipe/oci/driver/UTestOCIObjectStorageSourceType.java b/drivers/src/test/java/org/gorpipe/oci/driver/UTestOCIObjectStorageSourceType.java
@@ -1,5 +1,6 @@
 package org.gorpipe.oci.driver;
 
+import org.gorpipe.base.security.Credentials;
 import org.gorpipe.gor.driver.GorDriverFactory;
 import org.gorpipe.gor.driver.meta.SourceReference;
 import org.gorpipe.utils.DriverUtils;
@@ -92,7 +93,10 @@ public void testResolveS3HttpUrl() throws IOException {
     @Test
     public void testResolveS3OciHttpUrl() throws IOException {
         var source = GorDriverFactory.fromConfig().resolveDataSource(
-                new SourceReference("s3://namespace.objectstorage.us-ashburn-1.oci.customer-oci.com/the/path.dat", null, null, null, null,  false));
+                new SourceReference("s3://namespace.objectstorage.us-ashburn-1.oci.customer-oci.com/the/path.dat",
+                        DriverUtils.createSecurityContext("s3", "namespace.objectstorage.us-ashburn-1.oci.customer-oci.com",
+                               Credentials.OwnerType.System, "", "dummy", "dummy",
+                                "", ""), null, null, null,  false));
         assertNotEquals(OCIObjectStorageSourceType.OCI_OBJECT_STORAGE, source.getSourceType());
     }