Helm basic auth

helm-wrapper/src/main/java/io/github/inseefrlab/helmwrapper/service/HelmRepoService.java

@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.github.inseefrlab.helmwrapper.model.HelmRepo;
 import io.github.inseefrlab.helmwrapper.utils.Command;
+import io.micrometer.common.util.StringUtils;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.concurrent.TimeoutException;
@@ -28,7 +29,9 @@ public String addHelmRepo(
             final String url,
             final String nomRepo,
             final boolean skipTlsVerify,
-            final String caFile)
+            final String caFile,
+            String username,
+            String password)
             throws InvalidExitValueException, IOException, InterruptedException, TimeoutException {
         String command = "helm repo add ";
         if (skipTlsVerify) {
@@ -39,6 +42,12 @@ public String addHelmRepo(
                             "--ca-file " + System.getenv("CACERTS_DIR") + "/" + caFile + " ");
         }
         command = command.concat(nomRepo + " " + url);
+        if (StringUtils.isNotEmpty(username)) {
+            command = command.concat("--username " + username + " ");
+        }
+        if (StringUtils.isNotEmpty(password)) {
+            command = command.concat("--password " + password + " ");
+        }
         return Command.execute(command).getOutput().getString();
     }
 

helm-wrapper/src/test/java/io/github/inseefrlab/helmwrapper/RepoServiceTest.java

@@ -18,7 +18,7 @@ public class RepoServiceTest {
     public void should() throws Exception {
         HelmRepoService repoService = new HelmRepoService();
         helmRepoService.addHelmRepo(
-                "https://inseefrlab.github.io/helm-charts", "inseefrlab", false, null);
+                "https://inseefrlab.github.io/helm-charts", "inseefrlab", false, null, null, null);
         HelmRepo[] repos = helmRepoService.getHelmRepo();
         Assertions.assertEquals(1, repos.length);
     }

onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java

@@ -92,6 +92,12 @@ public class CatalogWrapper {
     @Schema(description = "If multipleServicesMode is set to maxNumber, how many versions to keep")
     private int maxNumberOfVersions = 5;
 
+    @Schema(description = "Username for basic authentication")
+    private String username = null;
+
+    @Schema(description = "Password for basic authentication")
+    private String password = null;
+
     public enum MultipleServicesMode {
         @JsonProperty("all")
         ALL,
@@ -304,6 +310,22 @@ public void setRestrictions(List<CatalogRestrictions> restrictions) {
         this.restrictions = restrictions;
     }
 
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
     public static class CatalogRestrictions {
 
         private UserAttribute userAttribute;

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java

@@ -13,9 +13,14 @@
 import fr.insee.onyxia.model.catalog.Pkg;
 import fr.insee.onyxia.model.helm.Chart;
 import fr.insee.onyxia.model.helm.Repository;
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.*;
 import java.util.stream.Collectors;
+import okhttp3.Credentials;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
@@ -26,7 +31,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.core.io.Resource;
 import org.springframework.core.io.ResourceLoader;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -40,10 +44,15 @@ public class CatalogLoader {
     private final ResourceLoader resourceLoader;
     private final ObjectMapper mapperHelm;
 
+    private OkHttpClient httpClient;
+
     public CatalogLoader(
-            ResourceLoader resourceLoader, @Qualifier("helm") ObjectMapper mapperHelm) {
+            ResourceLoader resourceLoader,
+            @Qualifier("helm") ObjectMapper mapperHelm,
+            OkHttpClient httpClient) {
         this.resourceLoader = resourceLoader;
         this.mapperHelm = mapperHelm;
+        this.httpClient = httpClient;
     }
 
     public void updateCatalog(CatalogWrapper cw) {
@@ -57,14 +66,11 @@ public void updateCatalog(CatalogWrapper cw) {
 
     /** TODO : move this helm specific logic somewhere else ? */
     private void updateHelmRepository(CatalogWrapper cw) {
-        try {
-            Reader reader =
-                    new InputStreamReader(
-                            resourceLoader
-                                    .getResource(cw.getLocation() + "/index.yaml")
-                                    .getInputStream(),
-                            UTF_8);
-            Repository repository = mapperHelm.readValue(reader, Repository.class);
+
+        try (InputStream stream =
+                fetchResource(
+                        cw.getLocation() + "/index.yaml", cw.getUsername(), cw.getPassword())) {
+            Repository repository = mapperHelm.readValue(stream, Repository.class);
 
             repository.setEntries(
                     repository.getEntries().entrySet().stream()
@@ -120,6 +126,19 @@ private void updateHelmRepository(CatalogWrapper cw) {
         }
     }
 
+    private InputStream fetchResource(String url, String username, String password)
+            throws IOException {
+        if (url.startsWith("http")) {
+            Request.Builder builder = new Request.Builder().url(url);
+            if (username != null && password != null) {
+                builder = builder.addHeader("Authorization", Credentials.basic(username, password));
+            }
+            return httpClient.newCall(builder.build()).execute().body().byteStream();
+        } else {
+            return resourceLoader.getResource(url).getInputStream();
+        }
+    }
+
     private void epurateChartsList(List<Chart> charts, CatalogWrapper cw) {
         if (cw.getMultipleServicesMode() == CatalogWrapper.MultipleServicesMode.ALL) {
             return;
@@ -193,13 +212,12 @@ private void refreshPackage(CatalogWrapper cw, Pkg pkg)
             absoluteUrl = StringUtils.applyRelativePath(cw.getLocation() + "/", chartUrl);
         }
 
-        Resource resource = resourceLoader.getResource(absoluteUrl);
-
-        try (InputStream inputStream = resource.getInputStream()) {
+        try (InputStream inputStream =
+                fetchResource(absoluteUrl, cw.getUsername(), cw.getPassword())) {
             extractDataFromTgz(inputStream, chart);
         } catch (IOException e) {
             throw new CatalogLoaderException(
-                    "Exception occurred during loading resource: " + resource.getDescription(), e);
+                    "Exception occurred during loading resource: " + absoluteUrl, e);
         }
     }
 

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogRefresher.java

@@ -39,7 +39,9 @@ private void refreshCatalogs() {
                                         c.getLocation(),
                                         c.getId(),
                                         c.getSkipTlsVerify(),
-                                        c.getCaFile());
+                                        c.getCaFile(),
+                                        c.getUsername(),
+                                        c.getPassword());
                                 LOGGER.info("Updating catalog: {}", c.getId());
                                 catalogLoader.updateCatalog(c);
                             } catch (Exception e) {

onyxia-api/src/main/resources/catalogs.json

@@ -24,6 +24,8 @@
         "user": true,
         "project": true
       },
+      "username": null,
+      "password": null,
       "multipleServicesMode" : "latest"
     },
     {
@@ -51,6 +53,8 @@
         "user": true,
         "project": true
       },
+      "username": null,
+      "password": null,
       "multipleServicesMode" : "latest"
     },
     {
@@ -77,6 +81,8 @@
         "user": true,
         "project": true
       },
+      "username": null,
+      "password": null,
       "multipleServicesMode" : "latest"
     }
   ]

onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java

@@ -1,22 +1,12 @@
 package fr.insee.onyxia.api.dao.universe;
 
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.is;
-import static org.junit.jupiter.api.Assertions.*;
-import static org.junit.jupiter.params.provider.Arguments.argumentSet;
-
 import fr.insee.onyxia.api.configuration.CatalogWrapper;
 import fr.insee.onyxia.api.configuration.CustomObjectMapper;
+import fr.insee.onyxia.api.configuration.HttpClientProvider;
 import fr.insee.onyxia.api.services.JsonSchemaRegistryService;
 import fr.insee.onyxia.api.services.JsonSchemaResolutionService;
 import fr.insee.onyxia.api.util.TestUtils;
 import fr.insee.onyxia.model.helm.Chart;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -29,13 +19,26 @@
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.springframework.util.CollectionUtils;
 
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.params.provider.Arguments.argumentSet;
+
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(
         classes = {
             CatalogLoader.class,
             CustomObjectMapper.class,
             JsonSchemaResolutionService.class,
-            JsonSchemaRegistryService.class
+            JsonSchemaRegistryService.class,
+            HttpClientProvider.class
         })
 public class CatalogLoaderTest {
 
@@ -151,13 +154,11 @@ void packageOnClassPathNotFound() {
         catalogLoader.updateCatalog(cw);
 
         String stdErrLogs = TestUtils.tapSystemOut(() -> catalogLoader.updateCatalog(cw));
-
         assertThat(
                 stdErrLogs,
                 containsString(
                         "fr.insee.onyxia.api.dao.universe.CatalogLoaderException: "
-                                + "Exception occurred during loading resource: class path resource "
-                                + "[catalog-loader-test/keepeme1.gz]"));
+                                + "Exception occurred during loading resource: classpath"));
     }
 
     @ParameterizedTest

onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogRefresherTest.java

@@ -82,12 +82,13 @@ void testSuccessfulRefresh() throws Exception {
         when(catalogWrapper.getCaFile()).thenReturn(null);
 
         when(catalogs.getCatalogs()).thenReturn(List.of(catalogWrapper));
-        when(helmRepoService.addHelmRepo("location", "id", false, null)).thenReturn("Repo added");
+        when(helmRepoService.addHelmRepo("location", "id", false, null, null, null))
+                .thenReturn("Repo added");
 
         catalogRefresher.run();
 
         verify(helmRepoService).repoUpdate();
-        verify(helmRepoService, times(1)).addHelmRepo("location", "id", false, null);
+        verify(helmRepoService, times(1)).addHelmRepo("location", "id", false, null, null, null);
         verify(catalogLoader, times(1)).updateCatalog(catalogWrapper);
     }
 }