PO-282: Automatically create required locations on Azure (#348)

build.gradle

@@ -190,7 +190,7 @@ sonarqube {
  property "sonar.projectKey", "uk.gov.hmcts:opal-fines-service"
  property "sonar.gradle.skipCompile", "true"
  property "sonar.exclusions", coverageExclusions.join(', ')
- property 'sonar.coverage.exclusions', "**/entity/*,**/model/*,**/exception/*,**/sftp/*,**/repository/jpa/*,**/opal/dto/*"
+ property 'sonar.coverage.exclusions', "**/entity/*,**/model/*,**/exception/*,**/config/**,**/repository/jpa/*,**/opal/dto/*"
  }
 }
 

charts/opal-fines-service/values.stg.template.yaml

@@ -6,4 +6,6 @@ java:
  OPAL_FRONTEND_URL: https://opal-frontend-staging.staging.platform.hmcts.net
  OPAL_LEGACY_GATEWAY_URL: https://opal-legacy-db-stub.staging.platform.hmcts.net
  TESTING_SUPPORT_ENDPOINTS_ENABLED: true
+ OPAL_SFTP_INBOUND_CREATE_SUB_LOCATIONS: true
+ OPAL_SFTP_OUTBOUND_CREATE_SUB_LOCATIONS: true
 

src/main/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJob.java

@@ -10,7 +10,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import uk.gov.hmcts.opal.scheduler.model.CronJob;
-import uk.gov.hmcts.opal.sftp.SftpService;
+import uk.gov.hmcts.opal.sftp.SftpOutboundService;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,7 +28,7 @@ public class FileHandlerJob implements CronJob {
  private String cronExpression;
 
  @Autowired
- private SftpService sftpService;
+ private SftpOutboundService sftpOutboundService;
 
 
  @SneakyThrows
@@ -40,7 +40,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
  String fileName = format("test-file-%s.txt", now());
  this.uploadFile("My file contents here...", fileName);
 
- sftpService.downloadOutboundFile("", fileName, this::logInputStream);
+ sftpOutboundService.downloadFile("", fileName, this::logInputStream);
 
  log.info(
  "Job ** {} ** completed. Next job scheduled @ {}",
@@ -51,7 +51,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
  }
 
  public void uploadFile(String contents, String fileName) {
- sftpService.uploadOutboundFile(format("%s %s", contents, now()).getBytes(), "", fileName);
+ sftpOutboundService.uploadFile(format("%s %s", contents, now()).getBytes(), "", fileName);
  }
 
 

src/main/java/uk/gov/hmcts/opal/sftp/SftpDirection.java

@@ -0,0 +1,7 @@
+package uk.gov.hmcts.opal.sftp;
+
+public enum SftpDirection {
+ INBOUND,
+ OUTBOUND
+}
+

src/main/java/uk/gov/hmcts/opal/sftp/SftpInboundService.java

@@ -0,0 +1,45 @@
+package uk.gov.hmcts.opal.sftp;
+
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.stereotype.Service;
+import uk.gov.hmcts.opal.sftp.config.SftpConnection;
+
+import java.io.InputStream;
+import java.util.function.Consumer;
+
+import static uk.gov.hmcts.opal.sftp.SftpLocation.getInboundLocations;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class SftpInboundService {
+
+ private final DefaultSftpSessionFactory inboundSessionFactory;
+ private final SftpService sftpService;
+ private final SftpConnection inboundConnection;
+
+ public void uploadFile(byte[] fileBytes, String path, String fileName) {
+ sftpService.uploadFile(inboundSessionFactory, fileBytes, path, fileName);
+ }
+
+ public boolean downloadFile(String path, String fileName, Consumer<InputStream> fileProcessor) {
+ return sftpService.downloadFile(inboundSessionFactory, path, fileName, fileProcessor);
+ }
+
+ public boolean deleteFile(String path, String fileName) {
+ return sftpService.deleteFile(inboundSessionFactory, path, fileName);
+ }
+
+ @PostConstruct
+ public void createSftpLocations() {
+ if (inboundConnection.isCreateSubLocations()) {
+ getInboundLocations()
+ .forEach(sftpLocation ->
+ this.sftpService.createDirectoryIfNotExists(inboundSessionFactory, sftpLocation)
+ );
+ }
+ }
+}

src/main/java/uk/gov/hmcts/opal/sftp/SftpLocation.java

@@ -0,0 +1,48 @@
+package uk.gov.hmcts.opal.sftp;
+
+import lombok.Getter;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static uk.gov.hmcts.opal.sftp.SftpDirection.INBOUND;
+import static uk.gov.hmcts.opal.sftp.SftpDirection.OUTBOUND;
+
+@Getter
+public enum SftpLocation {
+
+ AUTO_CHEQUES(INBOUND, "auto-cheque", "Comes from OAGS (pushed)"),
+ AUTO_CASH(INBOUND, "auto-cash", "Comes from OAGS (pushed)"),
+ NATWEST(INBOUND, "natwest", "Comes from BAIS (pulled)"),
+ ALL_PAY_BT_BARCLAY_CARD(INBOUND, "allpay", "Comes from BAIS (pulled)"),
+ DWP_BAILIFFS(INBOUND, "dwp-bailiffs", "Comes from BAIS (pulled)"),
+
+ ALL_PAY(OUTBOUND, "allpay", "Goes to BAIS (pushed)"),
+ ARCHIVE(OUTBOUND, "allpay-archive", "Goes to OAGS (pushed)");
+
+ private final SftpDirection direction;
+ private final String path;
+ private final String description;
+
+ SftpLocation(SftpDirection direction, String path, String description) {
+ this.direction = direction;
+ this.path = path;
+ this.description = description;
+ }
+
+ public static List<SftpLocation> getLocations(SftpDirection direction) {
+ return Arrays.stream(SftpLocation.values())
+ .filter(location -> location.getDirection() == direction)
+ .collect(Collectors.toList());
+ }
+
+ public static List<SftpLocation> getInboundLocations() {
+ return getLocations(INBOUND);
+ }
+
+ public static List<SftpLocation> getOutboundLocations() {
+ return getLocations(OUTBOUND);
+ }
+
+}

src/main/java/uk/gov/hmcts/opal/sftp/SftpOutboundService.java

@@ -0,0 +1,45 @@
+package uk.gov.hmcts.opal.sftp;
+
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.stereotype.Service;
+import uk.gov.hmcts.opal.sftp.config.SftpConnection;
+
+import java.io.InputStream;
+import java.util.function.Consumer;
+
+import static uk.gov.hmcts.opal.sftp.SftpLocation.getOutboundLocations;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class SftpOutboundService {
+
+ private final DefaultSftpSessionFactory outboundSessionFactory;
+ private final SftpService sftpService;
+ private final SftpConnection outboundConnection;
+
+ public void uploadFile(byte[] fileBytes, String path, String fileName) {
+ sftpService.uploadFile(outboundSessionFactory, fileBytes, path, fileName);
+ }
+
+ public boolean downloadFile(String path, String fileName, Consumer<InputStream> fileProcessor) {
+ return sftpService.downloadFile(outboundSessionFactory, path, fileName, fileProcessor);
+ }
+
+ public boolean deleteFile(String path, String fileName) {
+ return sftpService.deleteFile(outboundSessionFactory, path, fileName);
+ }
+
+ @PostConstruct
+ public void createSftpLocations() {
+ if (outboundConnection.isCreateSubLocations()) {
+ getOutboundLocations()
+ .forEach(sftpLocation ->
+ this.sftpService.createDirectoryIfNotExists(outboundSessionFactory, sftpLocation)
+ );
+ }
+ }
+}

src/main/java/uk/gov/hmcts/opal/sftp/SftpService.java

@@ -1,29 +1,22 @@
 package uk.gov.hmcts.opal.sftp;
 
-import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.integration.file.remote.RemoteFileTemplate;
 import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.integration.sftp.session.SftpSession;
 import org.springframework.stereotype.Service;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.function.Consumer;
 
 import static java.lang.String.format;
 
 @Slf4j
 @Service
-@RequiredArgsConstructor
 public class SftpService {
 
- private final DefaultSftpSessionFactory inboundSessionFactory;
- private final DefaultSftpSessionFactory outboundSessionFactory;
-
- public void uploadOutboundFile(byte[] fileBytes, String path, String fileName) {
- uploadFile(outboundSessionFactory, fileBytes, path, fileName);
- }
-
  public void uploadFile(DefaultSftpSessionFactory sessionFactory, byte[] fileBytes, String path, String fileName) {
  var template = new RemoteFileTemplate<>(sessionFactory);
  template.execute(session -> {
@@ -33,14 +26,6 @@ public void uploadFile(DefaultSftpSessionFactory sessionFactory, byte[] fileByte
  });
  }
 
- public boolean downloadInboundFile(String path, String fileName, Consumer<InputStream> fileProcessor) {
- return downloadFile(inboundSessionFactory, path, fileName, fileProcessor);
- }
-
- public boolean downloadOutboundFile(String path, String fileName, Consumer<InputStream> fileProcessor) {
- return downloadFile(outboundSessionFactory, path, fileName, fileProcessor);
- }
-
  public boolean downloadFile(DefaultSftpSessionFactory sessionFactory,
  String path,
  String fileName,
@@ -49,17 +34,38 @@ public boolean downloadFile(DefaultSftpSessionFactory sessionFactory,
  return template.get(path + "/" + fileName, fileProcessor::accept);
  }
 
- public boolean deleteOutboundFile(String path, String fileName) {
- return deleteFile(outboundSessionFactory, path, fileName);
+ public boolean deleteFile(DefaultSftpSessionFactory sessionFactory, String path, String fileName) {
+ var template = new RemoteFileTemplate<>(sessionFactory);
+ return template.execute(session -> session.remove(path + "/" + fileName));
  }
 
- public boolean deleteInboundFile(String path, String fileName) {
- return deleteFile(inboundSessionFactory, path, fileName);
+ public void createDirectoryIfNotExists(DefaultSftpSessionFactory sessionFactory, SftpLocation location) {
+ try {
+ String remoteDirectory = location.getPath();
+ if (!directoryExists(sessionFactory, remoteDirectory)) {
+ sessionFactory.getSession().mkdir(remoteDirectory);
+ log.info(format(
+ "%s SFTP directory %s created for %s",
+ location.getDirection(),
+ remoteDirectory,
+ location.getDescription()
+ ));
+ }
+ } catch (Exception exception) {
+ log.error(exception.getMessage(), exception);
+ }
  }
 
- public boolean deleteFile(DefaultSftpSessionFactory sessionFactory, String path, String fileName) {
- var template = new RemoteFileTemplate<>(sessionFactory);
- return template.execute(session -> session.remove(path + "/" + fileName));
+ public boolean directoryExists(DefaultSftpSessionFactory sessionFactory, String path) throws IOException {
+
+ try (SftpSession session = sessionFactory.getSession()) {
+ for (var file : session.list("/")) {
+ if (path.equalsIgnoreCase(file.getFilename())) {
+ return true;
+ }
+ }
+ return false;
+ }
  }
 
 }

src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConfiguration.java

@@ -11,25 +11,35 @@ public class SftpConfiguration {
 
  private final SftpProperties sftpProperties;
 
+ @Bean
+ public SftpConnection inboundConnection() {
+ return sftpProperties.getInbound();
+ }
+
  @Bean
  public DefaultSftpSessionFactory inboundSessionFactory() {
  DefaultSftpSessionFactory sessionFactory = new DefaultSftpSessionFactory();
- sessionFactory.setHost(sftpProperties.getInbound().getHost());
- sessionFactory.setPort(sftpProperties.getInbound().getPort());
- sessionFactory.setUser(sftpProperties.getInbound().getUser());
- sessionFactory.setPassword(sftpProperties.getInbound().getPassword());
+ sessionFactory.setHost(inboundConnection().getHost());
+ sessionFactory.setPort(inboundConnection().getPort());
+ sessionFactory.setUser(inboundConnection().getUser());
+ sessionFactory.setPassword(inboundConnection().getPassword());
  sessionFactory.setAllowUnknownKeys(true);
 
  return sessionFactory;
  }
 
+ @Bean
+ public SftpConnection outboundConnection() {
+ return sftpProperties.getOutbound();
+ }
+
  @Bean
  public DefaultSftpSessionFactory outboundSessionFactory() {
  DefaultSftpSessionFactory sessionFactory = new DefaultSftpSessionFactory();
- sessionFactory.setHost(sftpProperties.getOutbound().getHost());
- sessionFactory.setPort(sftpProperties.getOutbound().getPort());
- sessionFactory.setUser(sftpProperties.getOutbound().getUser());
- sessionFactory.setPassword(sftpProperties.getOutbound().getPassword());
+ sessionFactory.setHost(outboundConnection().getHost());
+ sessionFactory.setPort(outboundConnection().getPort());
+ sessionFactory.setUser(outboundConnection().getUser());
+ sessionFactory.setPassword(outboundConnection().getPassword());
  sessionFactory.setAllowUnknownKeys(true);
 
  return sessionFactory;

src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConnection.java

@@ -12,4 +12,5 @@ public class SftpConnection {
  private String user;
  private String password;
  private String location;
+ private boolean createSubLocations;
 }

src/main/resources/application.yaml

@@ -117,12 +117,14 @@ opal:
  user: ${OPAL_SFTP_INBOUND_USER:-}
  password: ${OPAL_SFTP_INBOUND_PASSWORD:-}
  location: ${OPAL_SFTP_INBOUND_LOCATION:inbound}
+ create-sub-locations: ${OPAL_SFTP_INBOUND_CREATE_SUB_LOCATIONS:false}
  outbound:
  host: ${OPAL_SFTP_OUTBOUND_HOST:opalsftpstg.blob.core.windows.net}
  port: ${OPAL_SFTP_OUTBOUND_PORT:22}
  user: ${OPAL_SFTP_OUTBOUND_USER:-}
  password: ${OPAL_SFTP_OUTBOUND_PASSWORD:-}
  location: ${OPAL_SFTP_OUTBOUND_LOCATION:outbound}
+ create-sub-locations: ${OPAL_SFTP_OUTBOUND_CREATE_SUB_LOCATIONS:false}
  schedule:
  log-retention-job:
  cron: ${OPAL_LOG_RETENTION_JOB_CRON:0 0 * * * ?}

src/test/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJobTest.java

@@ -11,7 +11,7 @@
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.slf4j.Logger;
-import uk.gov.hmcts.opal.sftp.SftpService;
+import uk.gov.hmcts.opal.sftp.SftpOutboundService;
 
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
@@ -33,7 +33,7 @@ class FileHandlerJobTest {
  JobExecutionContext jobExecutionContext;
 
  @Mock
- SftpService sftpService;
+ SftpOutboundService sftpOutboundService;
 
  @Mock
  Logger logger;
@@ -57,8 +57,8 @@ void testExecute() throws JobExecutionException {
 
  fileHandlerJob.execute(jobExecutionContext);
 
- verify(sftpService, times(1)).uploadOutboundFile(any(), anyString(), anyString());
- verify(sftpService, times(1)).downloadOutboundFile(anyString(), anyString(), any());
+ verify(sftpOutboundService, times(1)).uploadFile(any(), anyString(), anyString());
+ verify(sftpOutboundService, times(1)).downloadFile(anyString(), anyString(), any());
  }
 
  @Test