Initial S3 artifact synchronizer imlementation.

.gitignore

@@ -1,11 +1,32 @@
-target/
-pom.xml.tag
-pom.xml.releaseBackup
-pom.xml.versionsBackup
-pom.xml.next
-release.properties
-dependency-reduced-pom.xml
-buildNumber.properties
-.mvn/timing.properties
-# https://github.com/takari/maven-wrapper#usage-without-binary-jar
-.mvn/wrapper/maven-wrapper.jar
+# Compiled class file
+*.class
+target
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# ignore Intellij-IDEA files
+*.iml
+*.ipr
+*.iws
+*.settings
+*.project
+*.idea
+.DS_Store
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*

README.md

@@ -1 +1,20 @@
-# apim-artifact-sync-s3
+# Configure S3 as artifact synchronize mechanism for WSO2 API Manager
+
+With the WSO2 API Manager 3.2.0 onwards It's introduced a 3rd option to sync artifacts between gateways. In previous versions it supported only
+- Shared file system (e.g., NFS)
+- Rsync
+
+Now we have a extensible Inbuilt artifact synchronizer to sync artifacts. This is a sample implementation to share artifacts using the Amazon S3 service. 
+
+## Getting Started
+
+To get started, go to [WSO2 APIM 3.2.0 Sync Synapse Artifacts using S3 bucket]().
+
+## Build
+
+APIM 3.x version related code can be found in the master branch.
+
+Use the following command to build this implementation
+`mvn clean install`
+
+

pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Copyright (c) 2020, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+  ~
+  ~ WSO2 Inc. licenses this file to you under the Apache License,
+  ~ Version 2.0 (the "License"); you may not use this file except
+  ~ in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.wso2.example</groupId>
+    <artifactId>AmazonS3ArtifactSync</artifactId>
+    <version>1.0</version>
+    <name>AmazonS3ArtifactSync</name>
+    <packaging>bundle</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>1.11.253</version>
+        </dependency>
+        <dependency>
+            <groupId>org.wso2.carbon.apimgt</groupId>
+            <artifactId>org.wso2.carbon.apimgt.impl</artifactId>
+            <version>6.7.206</version>
+        </dependency>
+        <dependency>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.framework</artifactId>
+            <version>1.9.0</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.felix</groupId>
+                <artifactId>maven-bundle-plugin</artifactId>
+                <extensions>true</extensions>
+                <version>3.2.0</version>
+                <configuration>
+                    <instructions>
+                        <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
+                        <Bundle-Name>${project.artifactId}</Bundle-Name>
+                        <Import-Package>
+                            org.apache.commons.logging.*; version=1.0.4,
+                            *;resolution:=optional,
+                            org.wso2.carbon.apimgt.*,
+                            com.amazonaws.*; version=1.11.253,
+                        </Import-Package>
+                        <Export-Package>
+                            !org.wso2.caron.test.internal;
+                            org.wso2.example.*;
+                            com.amazonaws.*;
+                        </Export-Package>
+                        <DynamicImport-Package>*</DynamicImport-Package>
+                    </instructions>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

src/main/java/org/wso2/example/AWSUtils.java

@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2020, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+ *
+ * WSO2 Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.wso2.example;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.wso2.carbon.apimgt.impl.APIConstants;
+import org.wso2.carbon.apimgt.impl.APIManagerConfiguration;
+import org.wso2.carbon.apimgt.impl.gatewayartifactsynchronizer.exception.ArtifactSynchronizerException;
+import org.wso2.carbon.apimgt.impl.internal.ServiceReferenceHolder;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class AWSUtils {
+
+    private static final Log log = LogFactory.getLog(AWSUtils.class);
+    private static final String API_STATUS_KEY = "api-status";
+    private AmazonS3 s3Client;
+
+    public AWSUtils(AmazonS3 s3Client) {
+        this.s3Client = s3Client;
+    }
+
+    public String getFile(String bucketName, String fileObjKeyName, String gatewayInstruction)
+            throws ArtifactSynchronizerException {
+        try {
+            GetObjectRequest request = new GetObjectRequest(bucketName, fileObjKeyName);
+            S3Object object = s3Client.getObject(request);
+            if (object != null && (object.getObjectMetadata().getUserMetadata().get("gatewayInstruction")
+                    .equals(gatewayInstruction) ||
+                    APIConstants.GatewayArtifactSynchronizer.GATEWAY_INSTRUCTION_ANY.equals(gatewayInstruction))) {
+                InputStream inputStream = object.getObjectContent();
+                return getTextFromInputStream(inputStream);
+            } else {
+                return "";
+            }
+        } catch (AmazonServiceException e) {
+            throw new ArtifactSynchronizerException(e.getMessage(), e);
+        } catch (SdkClientException e) {
+            throw new ArtifactSynchronizerException(e.getMessage(), e);
+        } catch (IOException e) {
+            throw new ArtifactSynchronizerException("Error while reading S3 response.", e);
+        }
+    }
+
+    public List<String> getListOfAPIs(String bucket) throws IOException {
+        List<String> list = new ArrayList<>();
+        try {
+            ObjectListing listing = s3Client.listObjects(bucket);
+            List<S3ObjectSummary> summaries = listing.getObjectSummaries();
+            for (S3ObjectSummary key : summaries) {
+                String file = key.getKey();
+                S3Object object = s3Client.getObject(bucket, file);
+                if (object != null) {
+                    list.add(getTextFromInputStream(object.getObjectContent()));
+                }
+            }
+        } catch (AmazonServiceException e) {
+            log.error(e.getMessage(), e);
+        } catch (IOException e) {
+            log.error(e.getMessage(), e);
+        }
+        return list;
+    }
+
+    public void putAFile(String bucketName, String fileObjKeyName, InputStream input, ObjectMetadata metadata)
+            throws ArtifactSynchronizerException {
+        try {
+            PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, input, metadata);
+            s3Client.putObject(request);
+            log.debug("Object : " + fileObjKeyName + "Successfully created in " + bucketName);
+        } catch (AmazonServiceException e) {
+            throw new ArtifactSynchronizerException(e.getMessage(), e);
+        } catch (SdkClientException e) {
+            throw new ArtifactSynchronizerException(e.getMessage(), e);
+        }
+    }
+
+    public void createBucketIfNotExist(String bucket) throws ArtifactSynchronizerException {
+        CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucket);
+        try {
+            s3Client.createBucket(createBucketRequest);
+        } catch (AmazonS3Exception e) {
+            if (e.getStatusCode() == 409) {
+                log.warn("Bucket already exist.");
+            } else {
+                throw new ArtifactSynchronizerException(e.getMessage(), e);
+            }
+        }
+    }
+
+    private String getTextFromInputStream(InputStream input) throws IOException {
+        BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+        String line = null;
+        StringBuilder sb = new StringBuilder();
+        while ((line = reader.readLine()) != null) {
+            sb.append(line);
+        }
+        return sb.toString();
+    }
+
+    public boolean isPublishedInGateway(String bucketName, String fileObjKeyName, String status)
+            throws ArtifactSynchronizerException {
+        GetObjectRequest request = new GetObjectRequest(bucketName, fileObjKeyName);
+        try {
+            S3Object object = s3Client.getObject(request);
+            if (object != null) {
+                String gatewayInstruction = object.getObjectMetadata().getUserMetadata().get("gatewayInstruction");
+                if (gatewayInstruction.equals(status)) {
+                    return true;
+                }
+            }
+        } catch (AmazonS3Exception e) {
+            if (e.getStatusCode() == 404) {
+                log.warn("Requested file " + fileObjKeyName + " not exist in bucket : " + bucketName);
+            } else if (e.getStatusCode() == 400) {
+                log.warn("Requested bucket " + bucketName + " not exist.");
+            } else {
+                throw new ArtifactSynchronizerException(e.getMessage(), e);
+            }
+        }
+        return false;
+    }
+
+    public List<S3Object> getAllAPIs() {
+        APIManagerConfiguration apiManagerConfiguration = ServiceReferenceHolder.getInstance()
+                .getAPIManagerConfigurationService().getAPIManagerConfiguration();
+
+        List<S3ObjectSummary> fullS3ObjectSummarylist = new ArrayList<>();
+        for (String key : apiManagerConfiguration.getApiGatewayEnvironments().keySet()) {
+            String bucket = key.trim().replaceAll("\\s", "").toLowerCase();
+            ObjectListing listing = s3Client.listObjects(bucket);
+            List<S3ObjectSummary> summaries = listing.getObjectSummaries();
+            fullS3ObjectSummarylist = Stream.concat(fullS3ObjectSummarylist.stream(), summaries.stream())
+                    .collect(Collectors.toList());
+        }
+        List<S3Object> objectList = new ArrayList<>();
+        for (S3ObjectSummary summary : fullS3ObjectSummarylist) {
+            String file = summary.getKey();
+            S3Object object = s3Client.getObject(summary.getBucketName(), file);
+            objectList.add(object);
+        }
+        return objectList;
+    }
+}