Multiple data sources, image generation, s3 uploads

README.md

@@ -1,5 +1,10 @@
 # Football Updater
 
+Check for player match performance data from their latest game and generate images which contain their statistic for the selected game.
+
+These are uploaded to a S3 bucket and an email is sent to the configured email containing a generated post caption, S3 file links and Google Image search links for recent photos for the player.
+
+The players and teams to check are stored in RDS MySQL database and configured with an external cron job to run at scheduled times.
 
 ### Build
 To build jar use 
@@ -23,6 +28,53 @@ Additional env params can be passed in with
     --MAILER_TO_ADDRESS=****
     --ENDPOINT_SECRET=****
 
+### Configurable
+
+The main configurable groupings are:
+
+Sprint Datasource (Database)
+
+    SPRING_DATASOURCE_HOST
+    SPRING_DATASOURCE_NAME
+    SPRING_DATASOURCE_USERNAME
+    SPRING_DATASOURCE_PASSWORD
+
+Endpoint secret
+
+    ENDPOINT_SECRET
+
+Datasource (Stat sources)
+
+    DATASOURCE_PRIORITY
+
+Post Version
+
+    IG_POST_VERSION
+
+Stat Image generator
+
+    IMAGE_GENERATOR_ENABLED
+    IMAGE_GENERATOR_INPUT_PATH
+    IMAGE_GENERATOR_OUTPUT_PATH
+
+Mailer
+
+    MAILER_IS_ENABLED
+    MAILER_SUBJECT
+    MAILER_FROM_NAME
+    MAILER_FROM_ADDRESS
+    MAILER_FROM_PASSWORD
+    MAILER_TO_NAME
+    MAILER_TO_ADDRESS
+    MAILER_IS_ATTACH_IMAGES
+
+AWS S3 Uploads
+
+    AWS_S3_IS_ENABLED
+    AWS_ACCESS_KEY
+    AWS_SECRET_KEY
+    AWS_S3_BUCKET_NAME
+
 ### Local MYSQL
 
 Use to login to local mysql

build.gradle

@@ -5,7 +5,7 @@ plugins {
 }
 
 group = 'com.ko'
-version = '0.0.2-SNAPSHOT'
+version = '0.0.3-SNAPSHOT'
 
 java {
 	sourceCompatibility = '20'
@@ -30,6 +30,9 @@ dependencies {
 	implementation 'ch.qos.logback:logback-classic:1.4.8'
 	implementation 'org.jsoup:jsoup:1.15.3'
 	implementation 'org.simplejavamail:simple-java-mail:8.1.3'
+
+	implementation 'com.amazonaws:aws-java-sdk-s3:1.12.562'
+
 }
 
 springBoot {

src/main/java/com/ko/footballupdater/configuration/AmazonS3Properties.java

@@ -0,0 +1,46 @@
+package com.ko.footballupdater.configuration;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import jakarta.validation.constraints.NotNull;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Getter
+@Setter
+@Component
+@ConfigurationProperties(prefix = "aws.s3")
+public class AmazonS3Properties {
+
+    @NotNull
+    private boolean enabled;
+
+    @NotNull
+    private String accessKey;
+
+    @NotNull
+    private String secretKey;
+
+    @NotNull
+    private String bucketName;
+
+    @Bean
+    public AmazonS3 s3Client() {
+        // Set up S3 client
+        log.info("Initialising S3 client");
+        Regions clientRegion = Regions.AP_SOUTHEAST_2;
+        BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+        return AmazonS3ClientBuilder.standard()
+                .withRegion(clientRegion)
+                .withCredentials(new AWSStaticCredentialsProvider(credentials))
+                .build();
+    }
+}

src/main/java/com/ko/footballupdater/configuration/ImageGeneratorProperies.java

@@ -0,0 +1,24 @@
+package com.ko.footballupdater.configuration;
+
+import jakarta.validation.constraints.NotNull;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Getter
+@Setter
+@Component
+@ConfigurationProperties(prefix = "image.generator")
+public class ImageGeneratorProperies {
+
+    @NotNull
+    private boolean enabled;
+
+    @NotNull
+    private String inputPath;
+
+    @NotNull
+    private String outputPath;
+
+}

src/main/java/com/ko/footballupdater/configuration/InstagramPostProperies.java

@@ -0,0 +1,18 @@
+package com.ko.footballupdater.configuration;
+
+import jakarta.validation.constraints.NotNull;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Getter
+@Setter
+@Component
+@ConfigurationProperties(prefix = "ig.post")
+public class InstagramPostProperies {
+
+    @NotNull
+    private int version;
+
+}

src/main/java/com/ko/footballupdater/configuration/MailerProperties.java

@@ -12,9 +12,17 @@
 @ConfigurationProperties(prefix = "mailer")
 public class MailerProperties {
 
+    @NotNull
+    private boolean enabled;
+
     @NotNull
     private String subject;
+
     private MailerFromProperties from;
+
     private MailerToProperties to;
 
+    @NotNull
+    private boolean attachImages;
+
 }

src/main/java/com/ko/footballupdater/controllers/PlayerController.java

@@ -9,6 +9,7 @@
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -38,17 +39,27 @@ public class PlayerController {
     }
 
     @GetMapping(path="/data/update")
-    public @ResponseBody ResponseEntity<UpdatePlayersResponse> dataUpdate() {
-        UpdatePlayersResponse response = new UpdatePlayersResponse();
+    public @ResponseBody ResponseEntity<UpdatePlayersResponse> dataUpdateForAllPlayers() {
         try {
-            playerService.updateDataForAllPlayers(response);
-            return ResponseEntity.ok(response);
+            return ResponseEntity.ok(playerService.updateDataForAllPlayers());
         } catch (Exception ex) {
             throw new ResponseStatusException(
                     HttpStatus.BAD_REQUEST, "Updating players failed", ex);
         }
     }
 
+    @GetMapping(path="/data/update/{playerId}")
+    public @ResponseBody ResponseEntity<UpdatePlayersResponse> dataUpdateForPlayer(
+            @PathVariable("playerId") Integer playerId
+    ) {
+        try {
+            return ResponseEntity.ok(playerService.updateDataForPlayer(playerId));
+        } catch (Exception ex) {
+            throw new ResponseStatusException(
+                    HttpStatus.BAD_REQUEST, "Updating player failed", ex);
+        }
+    }
+
     @PostMapping(path="/data-source/update")
     public @ResponseBody DataSource updatePlayerDataSource(@RequestBody DataSource dataSource) {
         try {

src/main/java/com/ko/footballupdater/datasource/DataSourceParser.java

@@ -15,6 +15,6 @@ public interface DataSourceParser {
 
     PlayerMatchPerformanceStats parsePlayerMatchData(Player player, Document document);
 
-    List<Player> parseSquadDataForTeam(Team team, DataSource dataSource);
+    void parseSquadDataForTeam(Team team, DataSource dataSource, List<Player> players);
 
 }

src/main/java/com/ko/footballupdater/datasource/FbrefDataSource.java

@@ -77,9 +77,25 @@ public PlayerMatchPerformanceStats parsePlayerMatchData(Player player, Document
                 String latestMatchUrl = resultRow.select("th[data-stat=date] > a").attr("href");
 
                 // Check if match is new
-                if (player.getCheckedStatus() == null || (player.getCheckedStatus().getLatestCheckedMatchUrl() != null && player.getCheckedStatus().getLatestCheckedMatchUrl().equals(latestMatchUrl))) {
-                    // No new updates
-                    log.atInfo().setMessage(player.getName() + " " + "latestMatchUrl matches last checked").addKeyValue("player", player.getName()).log();
+                Date selectedMatchDate = null;
+                if (!resultRow.select("th[data-stat=date] > a").text().isEmpty()) {
+                    selectedMatchDate = dateFormat.parse(resultRow.select("th[data-stat=date] > a").text());
+                } else {
+                    log.atInfo().setMessage(player.getName() + " - Unable to get date from match row").addKeyValue("player", player.getName()).log();
+                    return null;
+                }
+
+                if (player.getCheckedStatus() != null) {
+                    if (player.getCheckedStatus().getLatestCheckedMatchDate() != null && !(selectedMatchDate.compareTo(player.getCheckedStatus().getLatestCheckedMatchDate()) > 0)) {
+                        log.atInfo().setMessage(player.getName() + " - Selected match is not newer than last checked").addKeyValue("player", player.getName()).log();
+                        return null;
+                    } else if (player.getCheckedStatus().getLatestCheckedMatchUrl() != null && player.getCheckedStatus().getLatestCheckedMatchUrl().equals(latestMatchUrl)) {
+                        // No new updates
+                        log.atInfo().setMessage(player.getName() + " - latestMatchUrl matches last checked").addKeyValue("player", player.getName()).log();
+                        return null;
+                    }
+                } else {
+                    log.atInfo().setMessage(player.getName() + " - CheckedStatus is null").addKeyValue("player", player.getName()).log();
                     return null;
                 }
 
@@ -93,12 +109,9 @@ public PlayerMatchPerformanceStats parsePlayerMatchData(Player player, Document
                     awayTeam = resultRow.select("td[data-stat=team] > a").text();
                     relevantTeam = awayTeam;
                 }
-                Date matchDate = null;
-                if (!resultRow.select("th[data-stat=date] > a").text().isEmpty()) {
-                    matchDate = dateFormat.parse(resultRow.select("th[data-stat=date] > a").text());
-                }
+
                 return new PlayerMatchPerformanceStats(
-                        new Match(latestMatchUrl, matchDate, homeTeam, awayTeam, relevantTeam),
+                        new Match(latestMatchUrl, selectedMatchDate, homeTeam, awayTeam, relevantTeam),
                         parseIntegerOrNull(resultRow.select("td[data-stat=minutes]").text()),
                         parseIntegerOrNull(resultRow.select("td[data-stat=goals]").text()),
                         parseIntegerOrNull(resultRow.select("td[data-stat=assists]").text()),
@@ -145,40 +158,46 @@ public PlayerMatchPerformanceStats parsePlayerMatchData(Player player, Document
     }
 
     @Override
-    public List<Player> parseSquadDataForTeam(Team team, DataSource dataSource) {
-        List<Player> players = new ArrayList<>();
+    public void parseSquadDataForTeam(Team team, DataSource dataSource, List<Player> players) {
         try {
             Document doc = Jsoup.connect(dataSource.getUrl()).get();
 
             Element tableElement = doc.getElementById("roster");
             if (tableElement == null) {
                 log.info("Unable to find roster/squad: table");
-                return null;
+                return;
             }
             Element tbodyElement = tableElement.getElementsByTag("tbody").first();
             if (tbodyElement == null) {
                 log.info("Cannot find any match results: tbody");
-                return null;
+                return;
             }
             Elements playerRows = tbodyElement.select("tr");
             if (playerRows.isEmpty()) {
                 log.info("Cannot find any match results in table: tr");
-                return null;
+                return;
             }
             for (Element playerRow : playerRows) {
-                Player player = new Player(
-                        playerRow.select("td[data-stat=player] > a").text(),
-                        dateFormat.parse(playerRow.select("td[data-stat=birth_date]").text())
-                );
-                Set<DataSource> dataSources = new HashSet<>();
-                dataSources.add(new DataSource(DataSourceType.PLAYER, DataSourceSiteName.FBREF, generatePlayerUrl(playerRow, player.getName())));
-                player.setDataSources(dataSources);
-                players.add(player);
+                String playerName = playerRow.select("td[data-stat=player] > a").text();
+                DataSource newDataSource = new DataSource(DataSourceType.PLAYER, DataSourceSiteName.FBREF, generatePlayerUrl(playerRow, playerName));
+                // Player exists in passed player list
+                if (players.stream().anyMatch(o -> o.getName().equals(playerName))) {
+                    Player existingPlayer = players.stream().filter(o -> o.getName().equals(playerName)).findFirst().get();
+                    existingPlayer.getDataSources().add(newDataSource);
+                } else {
+                    // Create player and add to list
+                    Player player = new Player(
+                            playerName,
+                            dateFormat.parse(playerRow.select("td[data-stat=birth_date]").text())
+                    );
+                    Set<DataSource> dataSources = new HashSet<>();
+                    dataSources.add(newDataSource);
+                    player.setDataSources(dataSources);
+                    players.add(player);
+                }
             }
-            return players;
         } catch (Exception ex) {
             log.warn("Unable to create players list to add: " + ex);
-            return null;
         }
     }