diff --git a/modules/siddhi-doc-gen/pom.xml b/modules/siddhi-doc-gen/pom.xml index 0a59ed7a8c..93cd9e6420 100644 --- a/modules/siddhi-doc-gen/pom.xml +++ b/modules/siddhi-doc-gen/pom.xml @@ -61,14 +61,6 @@ org.apache.commons commons-io - - org.json - json - - - org.jsoup - jsoup - diff --git a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/ExtensionDocCache.java b/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/ExtensionDocCache.java deleted file mode 100644 index 60f22e9f15..0000000000 --- a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/ExtensionDocCache.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * you may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.siddhi.doc.gen.extensions; - -import org.json.JSONObject; -import org.json.JSONTokener; - -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -/** - * The ExtensionDocCache class works as the main manipulator of underline - * API response cache. - */ -public class ExtensionDocCache { - - /** - * The description key in the cache. - */ - private static final String KEY_DESCRIPTION = "description"; - - /** - * The lastModifiedDateTime key in the cache. - */ - private static final String KEY_LAST_MODIFIED_DATETIME = "lastModifiedDateTime"; - - /** - * The actual JSON instance of the cache. - */ - private final JSONObject cache; - - /** - * A memento of the cache instance. - */ - private final JSONObject memento; - - /** - * The path to the cache in local storage. - */ - private final Path cachePath; - - /** - * Whether cache is in-memory or not. - */ - private boolean inMemory = false; - - /** - * Constructs a ExtensionDocCache instance. Any failure occurs while reading - * the cache from local storage will cause constructor to create an in-memory cache. - * - * @param cachePath the path to the cache - */ - public ExtensionDocCache(Path cachePath) { - JSONObject cache; - try { - JSONTokener tokener = new JSONTokener(cachePath.toUri().toURL().openStream()); - cache = new JSONObject(tokener); - if (cache.length() == 0) { - inMemory = true; - } - } catch (IOException e) { - inMemory = true; - cache = new JSONObject(); - } - this.cache = cache; - this.cachePath = cachePath; - memento = new JSONObject(cache.toString()); - } - - /** - * Return whether extension exists in the cache. - * - * @param extension the name of the cache - * @return whether extension exists in the cache - */ - public boolean has(String extension) { - return cache.has(extension); - } - - /** - * The add method adds an extension with description and last modified date to the cache. - * - * @param extension the name of the extension - * @param description the description for the extension - * @param lastModifiedDateTime the last modified date of the extension doc - */ - public void add(String extension, String description, String lastModifiedDateTime) { - JSONObject values = (cache.has(extension)) ? cache.getJSONObject(extension) : new JSONObject(); - - values.put(KEY_DESCRIPTION, description); - values.put(KEY_LAST_MODIFIED_DATETIME, lastModifiedDateTime); - - cache.put(extension, values); - } - - /** - * The getLastModifiedDateTime returns the last modified date - * a given extension. - * - * @param extension the name of the extension - * @return the the last modified date a given extension - */ - public String getLastModifiedDateTime(String extension) { - if (cache.has(extension)) { - JSONObject values = cache.getJSONObject(extension); - - return values.getString(KEY_LAST_MODIFIED_DATETIME); - } - return null; - } - - /** - * Remove an extension from the cache. - * - * @param extension the name of the extension - */ - public void remove(String extension) { - cache.remove(extension); - } - - /** - * Remove relative complement of the given extension set - * from the extension set in the cache. - * - * @param extensions the extension set - */ - public void removeComplementOf(Set extensions) { - cache.keySet().removeIf(e -> !extensions.contains(e)); - } - - /** - * @return whether cache is in-memory - */ - public boolean isInMemory() { - return inMemory; - } - - /** - * The commit method writes the data to the given location in the - * local storage. - * - * 1). If there is no update done to the {@link ExtensionDocCache#cache} - * it will return the false. - * 3). Due to any IO error if writing fails it will return the false. - * - * Overall method only returns true if it has written to the local storage. - * - * @return whether changes has made to the underline local storage - */ - public boolean commit() { - if (cache.equals(memento)) { - return false; - } - try { - try (PrintWriter writer = new PrintWriter(cachePath.toString(), StandardCharsets.UTF_8.toString())) { - writer.println(cache.toString(2)); - } - } catch (IOException e) { - return false; - } - return true; - } - - /** - * The getExtensionDescriptionMap method returns a map of extensions - * with extension name as key and extension description as value. - * - * @return a map of extensions with extension name as key and extension description as value. - */ - public Map getExtensionDescriptionMap() { - Map map = new TreeMap<>(); - Set extensions = cache.keySet(); - - for (String extension : extensions) { - JSONObject values = cache.getJSONObject(extension); - map.put(extension, values.getString(KEY_DESCRIPTION)); - } - - return map; - } -} diff --git a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/ExtensionDocRetriever.java b/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/ExtensionDocRetriever.java deleted file mode 100644 index 6ad1292890..0000000000 --- a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/ExtensionDocRetriever.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * you may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.siddhi.doc.gen.extensions; - -import io.siddhi.doc.gen.extensions.githubclient.ContentsBodyReader; -import io.siddhi.doc.gen.extensions.githubclient.ContentsResponse; -import io.siddhi.doc.gen.extensions.githubclient.GithubContentsClient; -import io.siddhi.doc.gen.extensions.githubclient.HtmlContentsResponse; -import org.apache.maven.plugin.logging.Log; -import org.apache.maven.plugin.logging.SystemStreamLog; - -import java.io.IOException; -import java.nio.file.attribute.FileTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -/** - * The ExtensionDocRetriever class provides ability to retrieve all the descriptions - * for Siddhi extensions. It manages process of retrieving and saving data from remote - * extension repositories. - */ -public class ExtensionDocRetriever { - - /** - * A log to log various kind of state changes. - */ - private static final Log log = new SystemStreamLog(); - - /** - * The Github username where all extensions exist. - */ - private final String baseGithubId; - - /** - * The extension names list. - */ - private final List extensions; - - private final ExtensionDocCache cache; - - /** - * Whether the API has reached it's limits. - */ - private boolean throttled = false; - - public ExtensionDocRetriever(String baseGithubId, List extensions, ExtensionDocCache cache) { - this.baseGithubId = baseGithubId; - this.extensions = extensions; - this.cache = cache; - } - - /** - * The pull method retrieves corresponding descriptions from extension repositories. - * The method only retrieves data if they have been modified in the repository. - * - * @return whether it is a successful pull. - */ - public boolean pull() { - try { - for (final String extension : extensions) { - GithubContentsClient githubClient = new GithubContentsClient.Builder(baseGithubId, extension) - .isReadme(true) - .build(); - if (cache.has(extension)) { - /* Sets HTTP header to make the request conditional */ - githubClient.setHeader("If-Modified-Since", cache.getLastModifiedDateTime(extension)); - } - HtmlContentsResponse response = githubClient.getContentsResponse(HtmlContentsResponse.class); - /* API throttling limit reached, could not process further without credentials */ - if (response.getStatus() == 403) { - throttled = true; - break; - } - updateCache(extension, response); - } - - cache.removeComplementOf(new TreeSet<>(extensions)); - - if (!throttled) { - cache.commit(); - } - } catch (IOException | ReflectiveOperationException e) { - return false; - } - return true; - } - - /** - * This updateCache method updates the underline cache according to API responses. - * - * @param extension the name of the extension - * @param response the API response. - * @throws IOException if error occurred while extracting the JSON type error. - */ - private void updateCache(String extension, HtmlContentsResponse response) throws IOException { - int status = response.getStatus(); - - switch (status) { - case 200: { - ContentsBodyReader reader = response.getContentsBodyReader(); - String firstParagraph = reader.getFirstParagraph(); - if (firstParagraph == null) { - return; - } - cache.add(extension, firstParagraph, response.getHeader("Last-Modified").get(0)); - break; - } - case 304: - break; - case 404: - cache.remove(extension); - break; - default: - log.error(String.format("Error occurred while retrieving the extension '%s': %d %s", - extension, status, response.getError().toString())); - } - } - - /** - * @return whether API has reached it's throttling limits. - */ - public boolean isThrottled() { - return throttled; - } -} diff --git a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/ContentsBodyReader.java b/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/ContentsBodyReader.java deleted file mode 100644 index 9af80ce4ef..0000000000 --- a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/ContentsBodyReader.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * you may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.siddhi.doc.gen.extensions.githubclient; - -/** - * The ContentsBodyReader class offers ability to read from a C type response body - * from a Github Contents API response. - * - * @param the response body type - */ -public abstract class ContentsBodyReader { - - /** - * The C type content. - */ - protected final C content; - - ContentsBodyReader(C content) { - this.content = content; - } - - /** - * Extract the first paragraph from the whole content body - * and return it as a String. - * - * @return the extracted paragraph - */ - public abstract String getFirstParagraph(); -} diff --git a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/ContentsResponse.java b/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/ContentsResponse.java deleted file mode 100644 index 47c57a0302..0000000000 --- a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/ContentsResponse.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * you may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.siddhi.doc.gen.extensions.githubclient; - -import org.apache.commons.io.IOUtils; -import org.json.JSONObject; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import javax.net.ssl.HttpsURLConnection; - -/** - * The ContentsResponse class represents an abstract view of the Github Contents API response. - * If the response is valid it will contain an instance of T type class representing response body. - * If the response is invalid it will contain an instance of JSONObject class representing the error. - * - * @param the type of the valid content body - */ -public abstract class ContentsResponse { - - /** - * Response body. - */ - final InputStream stream; - - /** - * HTTP status code in response. - */ - private final int status; - - private final Map> headers; - - /** - * Reader to read the response body. - */ - ContentsBodyReader contentsBodyReader; - - ContentsResponse(HttpsURLConnection connection) throws IOException { - connection.setRequestProperty("Accept", "application/vnd.githubclient.v3." + mediaType()); - - status = connection.getResponseCode(); - stream = (status == 200) ? connection.getInputStream() : connection.getErrorStream(); - - headers = connection.getHeaderFields(); - - contentsBodyReader = null; - } - - /** - * @return the media type of the body - */ - abstract String mediaType(); - - /** - * @return the body of the response - * @throws IOException if error occurs while parsing the response body as a type T instance - */ - public abstract T getContent() throws IOException; - - public List getHeader(String name) { - return headers.get(name); - } - - /** - * @return the ContentsBodyReader instance - */ - public ContentsBodyReader getContentsBodyReader() { - if (contentsBodyReader == null) { - throw new IllegalStateException(this.getClass().getCanonicalName() - + " does not implement a subclass of " - + ContentsBodyReader.class.getCanonicalName()); - } - return contentsBodyReader; - } - - /** - * Because of all API error responses are in JSON format the getError method always returns - * an instance of {@code org.json.JSONObject}. - * - * @return the error message as a JSONObject - * @throws IOException if error occurs while parsing the response body as a JSONObject - */ - public JSONObject getError() throws IOException { - if (status == 200) { - throw new IllegalStateException("Response does not contain an error."); - } - if (stream == null) { - return new JSONObject(); - } - return new JSONObject(IOUtils.toString(stream, "UTF-8")); - } - - /** - * @return the HTTP status code in the response. - */ - public int getStatus() { - return status; - } -} diff --git a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/GithubContentsClient.java b/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/GithubContentsClient.java deleted file mode 100644 index 9968c840bf..0000000000 --- a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/GithubContentsClient.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * you may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.siddhi.doc.gen.extensions.githubclient; - -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.net.URL; -import javax.net.ssl.HttpsURLConnection; - -/** - * The GithubContentsClient class offers the ability to make requests to fetch certain - * contents from Github using Github Contents REST API. - */ -public class GithubContentsClient { - - /** - * The domain of the API. - */ - private static final String API_DOMAIN = "api.github.com"; - - /** - * The HTTPS connection to the API. - */ - private final HttpsURLConnection connection; - - /** - * The GithubContentsClient builder. - */ - public static class Builder { - private final String owner; - private final String repos; - private final StringBuilder queryParamsBuilder; - - private boolean isReadme = false; - private String path = "/"; - - public Builder(String owner, String repos) { - this.owner = owner; - this.repos = repos; - queryParamsBuilder = new StringBuilder(); - } - - public Builder isReadme(boolean isReadme) { - this.isReadme = isReadme; - return this; - } - - public Builder path(String path) { - if (!path.isEmpty() && path.charAt(0) != '/') { - path = "/" + path; - } - this.path = path; - return this; - } - - public Builder queryParam(String key, String val) { - if (queryParamsBuilder.length() != 0) { - queryParamsBuilder.append("&"); - } - queryParamsBuilder.append(key).append("=").append(val); - return this; - } - - public GithubContentsClient build() throws IOException { - return new GithubContentsClient(this); - } - } - - private GithubContentsClient(Builder builder) throws IOException { - StringBuilder urlBuilder = new StringBuilder() - .append("https://") - .append(API_DOMAIN) - .append("/repos") - .append("/").append(builder.owner) - .append("/").append(builder.repos); - if (builder.isReadme) { - urlBuilder.append("/readme"); - } else { - urlBuilder.append("/contents").append(builder.path); - } - String queryParams = builder.queryParamsBuilder.toString(); - if (!queryParams.isEmpty()) { - urlBuilder.append("?").append(queryParams); - } - URL url = new URL(urlBuilder.toString()); - connection = (HttpsURLConnection) url.openConnection(); - } - - /** - * Sets a header in HTTPS request. - * - * @param key the name of the header - * @param val the value of the header - */ - public void setHeader(String key, String val) { - connection.setRequestProperty(key, val); - } - - /** - * The getContentsResponse method returns an instance of class T which extends the - * {@code ContentsResponse} class. - * - * @param tClass the Class instance of the type T - * @param the class which holds API response - * @return a instance of ContentsResponse - * @throws ReflectiveOperationException if tClass does not exist or - * if constructor fails to create an instance of T - */ - public T getContentsResponse(Class tClass) throws ReflectiveOperationException { - Constructor constructor = tClass.getDeclaredConstructor(HttpsURLConnection.class); - constructor.setAccessible(true); - return constructor.newInstance(connection); - } -} diff --git a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/HtmlContentsResponse.java b/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/HtmlContentsResponse.java deleted file mode 100644 index 23164c2d7e..0000000000 --- a/modules/siddhi-doc-gen/src/main/java/io/siddhi/doc/gen/extensions/githubclient/HtmlContentsResponse.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * you may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.siddhi.doc.gen.extensions.githubclient; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.io.IOException; -import javax.net.ssl.HttpsURLConnection; - -/** - * The HtmlContentsResponse class holds a response with a HTML type body. - */ -public class HtmlContentsResponse extends ContentsResponse { - - HtmlContentsResponse(HttpsURLConnection connection) throws IOException { - super(connection); - if (this.getStatus() != 200) { - super.contentsBodyReader = null; - } else { - super.contentsBodyReader = new ContentsBodyReader(this.getContent()) { - - @Override - public String getFirstParagraph() { - Elements pTags = super.content.getElementsByTag("p"); - if (pTags != null) { - Element firstPTag = pTags.first(); - if (firstPTag != null) { - return firstPTag.text(); - } - } - return null; - } - }; - } - } - - @Override - String mediaType() { - return "html"; - } - - @Override - public Document getContent() throws IOException { - return Jsoup.parse(super.stream, null, ""); - } -} diff --git a/pom.xml b/pom.xml index f9884b0c8e..0b3ada13d0 100644 --- a/pom.xml +++ b/pom.xml @@ -239,16 +239,6 @@ ${org.jacoco.version} test - - org.json - json - ${json.version} - - - org.jsoup - jsoup - ${jsoup.version} - @@ -791,8 +781,6 @@ 1.6.0 1.4 2.8 - 20180813 - 1.12.1 1.8 3.1.1