diff --git a/pom.xml b/pom.xml index 9d03204..2f55b0b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.perfwise.local.pluginsmanager jmeter-local-plugins-manager - 2.1 + 2.2 2.9.4 diff --git a/src/main/java/io/perfwise/local/pluginsmanager/controller/RestController.java b/src/main/java/io/perfwise/local/pluginsmanager/controller/RestController.java index de1f053..5864fa5 100644 --- a/src/main/java/io/perfwise/local/pluginsmanager/controller/RestController.java +++ b/src/main/java/io/perfwise/local/pluginsmanager/controller/RestController.java @@ -38,7 +38,6 @@ public RestController() { } public enum Plugins { - DEFAULT, PUBLIC, CUSTOM } @@ -137,7 +136,6 @@ public void loadRestApiServices() { get("/plugins-table", (req, res) -> { res.type("application/json"); - String type = req.queryParams("type"); PluginService pluginService = new PluginServiceImpl(); return pluginService.getPluginTable(); }); diff --git a/src/main/java/io/perfwise/local/pluginsmanager/scheduler/ScheduledTasks.java b/src/main/java/io/perfwise/local/pluginsmanager/scheduler/ScheduledTasks.java index 26998a8..7ce7f4e 100644 --- a/src/main/java/io/perfwise/local/pluginsmanager/scheduler/ScheduledTasks.java +++ b/src/main/java/io/perfwise/local/pluginsmanager/scheduler/ScheduledTasks.java @@ -29,7 +29,7 @@ public void run() { try { JSONArray pluginsArray = HttpRequest.get(props.getProperty("jmeter.plugins.url")); - if(getAvailablePluginsCount(pluginsArray) == 0){ + if(getAvailablePluginsCount() == 0){ LOGGER.info("Initializing plugin download for the fresh setup : {} New Plugin(s) found to download.. ", pluginsArray.length()); Parse.downloadAllPlugins(pluginsArray); }else{ @@ -50,8 +50,8 @@ public void run() { LOGGER.debug("Checking for plugin updates from Plugins manager"); } - private int getAvailablePluginsCount(JSONArray pluginsArray) throws SQLException, InterruptedException{ - int localStoreCount = Parse.getLocalPluginCount(pluginsArray); + private int getAvailablePluginsCount() throws SQLException, InterruptedException{ + int localStoreCount = Parse.getLocalPluginCount(); setLocalDBPluginsCount(localStoreCount); return localStoreCount; } diff --git a/src/main/java/io/perfwise/local/pluginsmanager/scheduler/http/HttpRequest.java b/src/main/java/io/perfwise/local/pluginsmanager/scheduler/http/HttpRequest.java index fd65594..0a1823c 100644 --- a/src/main/java/io/perfwise/local/pluginsmanager/scheduler/http/HttpRequest.java +++ b/src/main/java/io/perfwise/local/pluginsmanager/scheduler/http/HttpRequest.java @@ -1,8 +1,5 @@ package io.perfwise.local.pluginsmanager.scheduler.http; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import io.perfwise.local.pluginsmanager.model.MetadataModel; import io.perfwise.local.pluginsmanager.model.PluginModel; @@ -29,10 +26,7 @@ import java.net.*; import java.nio.file.Path; import java.nio.file.Paths; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -47,17 +41,18 @@ public class HttpRequest { private final String pluginsPath; private final String dependenciesPath; private final String customPluginsPath; + private final String mavenRepoUrl; private Connection conn; private static final HttpClient HTTP_CLIENT = HttpClients.createDefault(); private static final int MAX_RETRIES = 10; private static final long RETRY_INTERVAL_MS = 60000; private static final String INSERT_METADATA_INFO = "INSERT INTO metadata (id, version, changes, depends, downloadUrl, libs) VALUES (?, ?, ?, ?, ?, ?)"; private static final String INSERT_PLUGIN_INFO = "INSERT INTO plugins (id, name, type, description, helpUrl, markerClass, screenshotUrl, vendor, componentClasses, versions_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - private static final String SELECT_PLUGIN_VERSION_METADATA = "SELECT COUNT(*) AS COUNT FROM METADATA WHERE ID = ? AND VERSION = ?"; - private static final String SELECT_PLUGINS = "SELECT ID, NAME, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins"; + private static final String SELECT_PLUGINS_WITH_FILTER = "SELECT ID, NAME, TYPE, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins WHERE type = ?"; + private static final String SELECT_PLUGINS_WITHOUT_FILTER = "SELECT ID, NAME, TYPE, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins"; private static final String SELECT_PLUGINS_TABLE_DATA = "SELECT ID, NAME, TYPE, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES, VERSIONS_COUNT FROM plugins"; - private static final String SELECT_PLUGINS_WITH_FILTER = "SELECT ID, NAME, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins WHERE type = ?"; private static final String SELECT_METADATA_BY_ID = "SELECT ID, VERSION, CHANGES, DEPENDS, DOWNLOADURL, LIBS FROM metadata WHERE ID = ?"; + private static final String SELECT_PLUGIN_VERSION_METADATA = "SELECT COUNT(*) AS COUNT FROM METADATA WHERE ID = ? AND VERSION = ?"; public HttpRequest(Properties props){ @@ -66,6 +61,7 @@ public HttpRequest(Properties props){ this.pluginsPath = this.basePath.resolve("plugins").toString(); this.dependenciesPath = this.basePath.resolve("libs").toString(); this.customPluginsPath = this.basePath.resolve("custom").toString(); + this.mavenRepoUrl = props.getProperty("mvn.repo.url"); } public static JSONArray get(String url) throws IOException { @@ -138,10 +134,9 @@ public void downloadMissingPlugins(JSONObject pluginObject) throws URISyntaxExce }else{ List availableVersions = this.getAvailableLibraryVersions(downloadUrl); for (String ver : availableVersions){ - String url = downloadUrl.replace("%1$s", ver); - metaDataObj.put("downloadUrl", url.substring(url.lastIndexOf('/') + 1)); - fileDownloader(this.pluginsPath, new URI(url).toURL()); + fileDownloader(this.pluginsPath, new URI(ver).toURL()); } + metaDataObj.put("downloadUrl", getLibraryName(downloadUrl)+"-%1$s.jar"); } if(verObj.has("libs")){ JSONObject libsObject = verObj.getJSONObject("libs"); @@ -163,39 +158,49 @@ public void downloadMissingPlugins(JSONObject pluginObject) throws URISyntaxExce } private List getAvailableLibraryVersions(String downloadUrl) { + return fetchAvailableVersions(getLibraryName(downloadUrl)); + } + + private String getLibraryName(String downloadUrl) { String libName = null; Pattern pattern = Pattern.compile("jmeter/(?[^/]+)/%\\d[^/]*"); Matcher matcher = pattern.matcher(downloadUrl); if(matcher.find()) { libName = matcher.group("libName"); } - return fetchAvailableVersions(libName); + return libName; } - public JSONArray fetchPluginsFromLocalDB(String query, String type){ + public JSONArray fetchPluginsFromLocalDB(String query, String type) { JSONArray jsonArray = new JSONArray(); + ResultSet rs; + PreparedStatement preparedStatement= null; + Statement statement = null; try{ if(conn == null || conn.isClosed()){ conn = SQLiteConnectionPool.getConnection(); } - PreparedStatement preparedStatement = conn.prepareStatement(query); - if(type != null){ + if(!type.equals("all")){ + preparedStatement = conn.prepareStatement(query); preparedStatement.setString(1, type); + rs = preparedStatement.executeQuery(); + }else{ + statement = conn.createStatement(); + rs = statement.executeQuery(query); } - ResultSet rs = preparedStatement.executeQuery(); while (rs.next()) { JSONObject pluginObject = new JSONObject(); JSONObject libraryObj; - pluginObject.put("id", rs.getString("id")); pluginObject.put("name", rs.getString("name")); pluginObject.put("description", rs.getString("description")); pluginObject.put("helpUrl", rs.getString("helpUrl")); - pluginObject.put("markerClass", rs.getString("markerClass")); + String markerClass = rs.getString("markerClass"); + pluginObject.put("markerClass", (markerClass == null || markerClass.isEmpty()) ? JSONObject.NULL : markerClass); pluginObject.put("screenshotUrl", rs.getString("screenshotUrl")); pluginObject.put("vendor", rs.getString("vendor")); - libraryObj = this.getDependentLibraryObj(rs.getString("id"), type); + libraryObj = this.getDependentLibraryObj(rs.getString("id"), rs.getString("type")); pluginObject.put("versions", libraryObj); String componentClasses = rs.getString("componentClasses"); if(componentClasses != null){ @@ -203,29 +208,27 @@ public JSONArray fetchPluginsFromLocalDB(String query, String type){ } jsonArray.put(pluginObject); } - preparedStatement.close(); + rs.close(); }catch(SQLException | InterruptedException e){ LOGGER.error("Exception occurred while fetching plugins information"); } catch (UnknownHostException e) { throw new RuntimeException(e); } finally { + try{ + if(preparedStatement!=null) + preparedStatement.close(); + if(statement!=null) + statement.close(); + }catch (SQLException e){ + LOGGER.error("SQL Exception ", e); + } SQLiteConnectionPool.releaseConnection(conn); } return jsonArray; } public JSONArray getAllPlugins() { - JSONArray publicPluginArray = getPublicPlugins(); - JSONArray customPluginArray = getCustomPlugins(); - - for(Object obj: customPluginArray){ - publicPluginArray.put((JSONObject) obj); - } - return publicPluginArray; - } - - private JSONArray getCombinedPlugins() { - return fetchPluginsFromLocalDB(SELECT_PLUGINS, "all"); + return fetchPluginsFromLocalDB(SELECT_PLUGINS_WITHOUT_FILTER, "all"); } public JSONArray getPublicPlugins() { @@ -242,17 +245,18 @@ public JSONArray getAllPluginsTableData() { private JSONArray fetchAllPluginsTableData() { JSONArray jsonArray = new JSONArray(); + PreparedStatement preparedStatement; + ResultSet rs; + try{ if(conn == null || conn.isClosed()){ conn = SQLiteConnectionPool.getConnection(); } - PreparedStatement preparedStatement = conn.prepareStatement(HttpRequest.SELECT_PLUGINS_TABLE_DATA); - ResultSet rs = preparedStatement.executeQuery(); + preparedStatement = conn.prepareStatement(HttpRequest.SELECT_PLUGINS_TABLE_DATA); + rs = preparedStatement.executeQuery(); while (rs.next()) { JSONObject pluginObject = new JSONObject(); - JSONObject libraryObj; - pluginObject.put("id", rs.getString("id")); pluginObject.put("name", rs.getString("name")); pluginObject.put("type", rs.getString("type")); @@ -278,7 +282,7 @@ private JSONObject getDependentLibraryObj(String id, String type) throws Unknown String host = InetAddress.getLocalHost().getHostAddress(); String libUrl = String.format("http://%s:%s/%s/", host, props.getProperty("server.port"), "libs"); - String pluginUrl = null; + String pluginUrl; if(type.equals("public")){ pluginUrl = String.format("http://%s:%s/%s/", host, props.getProperty("server.port"), "plugins"); }else{ @@ -326,15 +330,15 @@ private JSONObject processLibs(String libs, String libUrl) { } private List fetchAvailableVersions(String libName) { - Document doc = null; + Document doc; List versions = new ArrayList<>(); try{ - doc = Jsoup.connect("https://repo1.maven.org/maven2/org/apache/jmeter/" + libName).get(); - Elements links = doc.select("a[href]"); + doc = Jsoup.connect(mavenRepoUrl + libName).get(); + Elements links = doc.select("a[title]"); for(Element link : links) { String href = link.attr("href"); - if(href.contains(libName + "-")) { - String version = href.substring(href.lastIndexOf("-") + 1, href.lastIndexOf(".jar")); + if(!href.contains("xml")){ + String version = mavenRepoUrl + libName + "/" + href + libName + "-" + href.substring(0, href.lastIndexOf('/')) + ".jar"; versions.add(version); } } @@ -394,8 +398,7 @@ public void updateCustomPluginInfoInDB(JSONObject pluginModel) { if(conn == null || conn.isClosed()){ conn = SQLiteConnectionPool.getConnection(); } - //"CREATE TABLE plugins (id TEXT PRIMARY KEY, name TEXT, type TEXT, description TEXT, helpUrl TEXT, markerClass TEXT, screenshotUrl TEXT, vendor TEXT, componentClasses TEXT, versions_count INTEGER)"; - //id, name, type, description, helpUrl, markerClass, screenshotUrl, vendor, componentClasses, versions_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + PreparedStatement preparedStatement = conn.prepareStatement(INSERT_PLUGIN_INFO); preparedStatement.setString(1, pluginModel.getString("id")); preparedStatement.setString(2, pluginModel.getString("name")); @@ -405,7 +408,7 @@ public void updateCustomPluginInfoInDB(JSONObject pluginModel) { preparedStatement.setString(6, pluginModel.getString("markerClass")); preparedStatement.setString(7, pluginModel.getString("screenshotUrl")); preparedStatement.setString(8, pluginModel.getString("vendor")); - preparedStatement.setString(9, pluginModel.getString("componentClasses")); + preparedStatement.setString(9, pluginModel.has("componentClasses") ? pluginModel.getString("componentClasses") : null); preparedStatement.setDouble(10, pluginModel.getDouble("version_count")); int rowsInserted = preparedStatement.executeUpdate(); diff --git a/src/main/java/io/perfwise/local/pluginsmanager/scheduler/parser/Parse.java b/src/main/java/io/perfwise/local/pluginsmanager/scheduler/parser/Parse.java index 1a2a61f..b399b4c 100644 --- a/src/main/java/io/perfwise/local/pluginsmanager/scheduler/parser/Parse.java +++ b/src/main/java/io/perfwise/local/pluginsmanager/scheduler/parser/Parse.java @@ -62,7 +62,7 @@ public static void downloadAllPlugins(JSONArray jmeterRepoJson) throws URISyntax LOGGER.info("Downloaded all the available plugins from the market"); } - public static int getLocalPluginCount(JSONArray pluginsArray) throws SQLException, InterruptedException { + public static int getLocalPluginCount() throws SQLException, InterruptedException { int localStoreCount = 0; conn = SQLiteConnectionPool.getConnection(); @@ -80,7 +80,7 @@ public static int getLocalPluginCount(JSONArray pluginsArray) throws SQLExceptio return localStoreCount; } - public static void addPluginDataToDB(JSONObject pluginObj) throws SQLException, InterruptedException { + public static void addPluginDataToDB(JSONObject pluginObj) { String type = "custom"; httpRequest.updateCustomPluginInfoInDB(pluginObj); } diff --git a/src/main/resources/config.properties b/src/main/resources/config.properties index 5b8eb10..d4e1790 100644 --- a/src/main/resources/config.properties +++ b/src/main/resources/config.properties @@ -11,7 +11,7 @@ db.timeout.secs=300 #External APIs jmeter.plugins.url=https://jmeter-plugins.org/repo/ -mvn.repo.url=https://mvnrepository.com/search?q= +mvn.repo.url=https://repo1.maven.org/maven2/org/apache/jmeter/ #Directory Configs #local.repo.path=/app/plugins-manager/