Merge pull request #6 from rollno748/ver1

Ver2

pom.xml

@@ -4,7 +4,7 @@
  <modelVersion>4.0.0</modelVersion>
  <groupId>io.perfwise.local.pluginsmanager</groupId>
  <artifactId>jmeter-local-plugins-manager</artifactId>
- <version>2.1</version>
+ <version>2.2</version>
 
  <properties>
  <spark-java>2.9.4</spark-java>

src/main/java/io/perfwise/local/pluginsmanager/controller/RestController.java

@@ -38,7 +38,6 @@ public RestController() {
  }
 
  public enum Plugins {
- DEFAULT,
  PUBLIC,
  CUSTOM
  }
@@ -137,7 +136,6 @@ public void loadRestApiServices() {
 
  get("/plugins-table", (req, res) -> {
  res.type("application/json");
- String type = req.queryParams("type");
  PluginService pluginService = new PluginServiceImpl();
  return pluginService.getPluginTable();
  });

src/main/java/io/perfwise/local/pluginsmanager/scheduler/ScheduledTasks.java

@@ -29,7 +29,7 @@ public void run() {
  try {
  JSONArray pluginsArray = HttpRequest.get(props.getProperty("jmeter.plugins.url"));
 
- if(getAvailablePluginsCount(pluginsArray) == 0){
+ if(getAvailablePluginsCount() == 0){
  LOGGER.info("Initializing plugin download for the fresh setup : {} New Plugin(s) found to download.. ", pluginsArray.length());
  Parse.downloadAllPlugins(pluginsArray);
  }else{
@@ -50,8 +50,8 @@ public void run() {
  LOGGER.debug("Checking for plugin updates from Plugins manager");
  }
 
- private int getAvailablePluginsCount(JSONArray pluginsArray) throws SQLException, InterruptedException{
- int localStoreCount = Parse.getLocalPluginCount(pluginsArray);
+ private int getAvailablePluginsCount() throws SQLException, InterruptedException{
+ int localStoreCount = Parse.getLocalPluginCount();
  setLocalDBPluginsCount(localStoreCount);
  return localStoreCount;
  }

src/main/java/io/perfwise/local/pluginsmanager/scheduler/http/HttpRequest.java

@@ -1,8 +1,5 @@
 package io.perfwise.local.pluginsmanager.scheduler.http;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.gson.Gson;
 import io.perfwise.local.pluginsmanager.model.MetadataModel;
 import io.perfwise.local.pluginsmanager.model.PluginModel;
@@ -29,10 +26,7 @@
 import java.net.*;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import java.sql.*;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
@@ -47,17 +41,18 @@ public class HttpRequest {
  private final String pluginsPath;
  private final String dependenciesPath;
  private final String customPluginsPath;
+ private final String mavenRepoUrl;
  private Connection conn;
  private static final HttpClient HTTP_CLIENT = HttpClients.createDefault();
  private static final int MAX_RETRIES = 10;
  private static final long RETRY_INTERVAL_MS = 60000;
  private static final String INSERT_METADATA_INFO = "INSERT INTO metadata (id, version, changes, depends, downloadUrl, libs) VALUES (?, ?, ?, ?, ?, ?)";
  private static final String INSERT_PLUGIN_INFO = "INSERT INTO plugins (id, name, type, description, helpUrl, markerClass, screenshotUrl, vendor, componentClasses, versions_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
- private static final String SELECT_PLUGIN_VERSION_METADATA = "SELECT COUNT(*) AS COUNT FROM METADATA WHERE ID = ? AND VERSION = ?";
- private static final String SELECT_PLUGINS = "SELECT ID, NAME, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins";
+ private static final String SELECT_PLUGINS_WITH_FILTER = "SELECT ID, NAME, TYPE, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins WHERE type = ?";
+ private static final String SELECT_PLUGINS_WITHOUT_FILTER = "SELECT ID, NAME, TYPE, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins";
  private static final String SELECT_PLUGINS_TABLE_DATA = "SELECT ID, NAME, TYPE, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES, VERSIONS_COUNT FROM plugins";
- private static final String SELECT_PLUGINS_WITH_FILTER = "SELECT ID, NAME, DESCRIPTION, HELPURL, MARKERCLASS, SCREENSHOTURL, VENDOR, COMPONENTCLASSES FROM plugins WHERE type = ?";
  private static final String SELECT_METADATA_BY_ID = "SELECT ID, VERSION, CHANGES, DEPENDS, DOWNLOADURL, LIBS FROM metadata WHERE ID = ?";
+ private static final String SELECT_PLUGIN_VERSION_METADATA = "SELECT COUNT(*) AS COUNT FROM METADATA WHERE ID = ? AND VERSION = ?";
 
 
  public HttpRequest(Properties props){
@@ -66,6 +61,7 @@ public HttpRequest(Properties props){
  this.pluginsPath = this.basePath.resolve("plugins").toString();
  this.dependenciesPath = this.basePath.resolve("libs").toString();
  this.customPluginsPath = this.basePath.resolve("custom").toString();
+ this.mavenRepoUrl = props.getProperty("mvn.repo.url");
  }
 
  public static JSONArray get(String url) throws IOException {
@@ -138,10 +134,9 @@ public void downloadMissingPlugins(JSONObject pluginObject) throws URISyntaxExce
  }else{
  List<String> availableVersions = this.getAvailableLibraryVersions(downloadUrl);
  for (String ver : availableVersions){
- String url = downloadUrl.replace("%1$s", ver);
- metaDataObj.put("downloadUrl", url.substring(url.lastIndexOf('/') + 1));
- fileDownloader(this.pluginsPath, new URI(url).toURL());
+ fileDownloader(this.pluginsPath, new URI(ver).toURL());
  }
+ metaDataObj.put("downloadUrl", getLibraryName(downloadUrl)+"-%1$s.jar");
  }
  if(verObj.has("libs")){
  JSONObject libsObject = verObj.getJSONObject("libs");
@@ -163,69 +158,77 @@ public void downloadMissingPlugins(JSONObject pluginObject) throws URISyntaxExce
  }
 
  private List<String> getAvailableLibraryVersions(String downloadUrl) {
+ return fetchAvailableVersions(getLibraryName(downloadUrl));
+ }
+
+ private String getLibraryName(String downloadUrl) {
  String libName = null;
  Pattern pattern = Pattern.compile("jmeter/(?<libName>[^/]+)/%\\d[^/]*");
  Matcher matcher = pattern.matcher(downloadUrl);
  if(matcher.find()) {
  libName = matcher.group("libName");
  }
- return fetchAvailableVersions(libName);
+ return libName;
  }
 
- public JSONArray fetchPluginsFromLocalDB(String query, String type){
+ public JSONArray fetchPluginsFromLocalDB(String query, String type) {
  JSONArray jsonArray = new JSONArray();
+ ResultSet rs;
+ PreparedStatement preparedStatement= null;
+ Statement statement = null;
  try{
  if(conn == null || conn.isClosed()){
  conn = SQLiteConnectionPool.getConnection();
  }
- PreparedStatement preparedStatement = conn.prepareStatement(query);
- if(type != null){
+ if(!type.equals("all")){
+  preparedStatement = conn.prepareStatement(query);
  preparedStatement.setString(1, type);
+ rs = preparedStatement.executeQuery();
+ }else{
+ statement = conn.createStatement();
+ rs = statement.executeQuery(query);
  }
- ResultSet rs = preparedStatement.executeQuery();
 
  while (rs.next()) {
  JSONObject pluginObject = new JSONObject();
  JSONObject libraryObj;
-
  pluginObject.put("id", rs.getString("id"));
  pluginObject.put("name", rs.getString("name"));
  pluginObject.put("description", rs.getString("description"));
  pluginObject.put("helpUrl", rs.getString("helpUrl"));
- pluginObject.put("markerClass", rs.getString("markerClass"));
+ String markerClass = rs.getString("markerClass");
+ pluginObject.put("markerClass", (markerClass == null || markerClass.isEmpty()) ? JSONObject.NULL : markerClass);
  pluginObject.put("screenshotUrl", rs.getString("screenshotUrl"));
  pluginObject.put("vendor", rs.getString("vendor"));
- libraryObj = this.getDependentLibraryObj(rs.getString("id"), type);
+ libraryObj = this.getDependentLibraryObj(rs.getString("id"), rs.getString("type"));
  pluginObject.put("versions", libraryObj);
  String componentClasses = rs.getString("componentClasses");
  if(componentClasses != null){
  pluginObject.put("componentClasses", new JSONArray(componentClasses));
  }
  jsonArray.put(pluginObject);
  }
- preparedStatement.close();
+ rs.close();
  }catch(SQLException | InterruptedException e){
  LOGGER.error("Exception occurred while fetching plugins information");
  } catch (UnknownHostException e) {
  throw new RuntimeException(e);
  } finally {
+ try{
+ if(preparedStatement!=null)
+ preparedStatement.close();
+ if(statement!=null)
+ statement.close();
+ }catch (SQLException e){
+ LOGGER.error("SQL Exception ", e);
+ }
  SQLiteConnectionPool.releaseConnection(conn);
  }
  return jsonArray;
  }
 
  public JSONArray getAllPlugins() {
- JSONArray publicPluginArray = getPublicPlugins();
- JSONArray customPluginArray = getCustomPlugins();
-
- for(Object obj: customPluginArray){
- publicPluginArray.put((JSONObject) obj);
- }
- return publicPluginArray;
- }
-
- private JSONArray getCombinedPlugins() {
- return fetchPluginsFromLocalDB(SELECT_PLUGINS, "all");
+ return fetchPluginsFromLocalDB(SELECT_PLUGINS_WITHOUT_FILTER, "all");
  }
 
  public JSONArray getPublicPlugins() {
@@ -242,17 +245,18 @@ public JSONArray getAllPluginsTableData() {
 
  private JSONArray fetchAllPluginsTableData() {
  JSONArray jsonArray = new JSONArray();
+ PreparedStatement preparedStatement;
+ ResultSet rs;
+
  try{
  if(conn == null || conn.isClosed()){
  conn = SQLiteConnectionPool.getConnection();
  }
- PreparedStatement preparedStatement = conn.prepareStatement(HttpRequest.SELECT_PLUGINS_TABLE_DATA);
- ResultSet rs = preparedStatement.executeQuery();
+ preparedStatement = conn.prepareStatement(HttpRequest.SELECT_PLUGINS_TABLE_DATA);
+ rs = preparedStatement.executeQuery();
 
  while (rs.next()) {
  JSONObject pluginObject = new JSONObject();
- JSONObject libraryObj;
-
  pluginObject.put("id", rs.getString("id"));
  pluginObject.put("name", rs.getString("name"));
  pluginObject.put("type", rs.getString("type"));
@@ -278,7 +282,7 @@ private JSONObject getDependentLibraryObj(String id, String type) throws Unknown
 
  String host = InetAddress.getLocalHost().getHostAddress();
  String libUrl = String.format("http://%s:%s/%s/", host, props.getProperty("server.port"), "libs");
- String pluginUrl = null;
+ String pluginUrl;
  if(type.equals("public")){
  pluginUrl = String.format("http://%s:%s/%s/", host, props.getProperty("server.port"), "plugins");
  }else{
@@ -326,15 +330,15 @@ private JSONObject processLibs(String libs, String libUrl) {
  }
 
  private List<String> fetchAvailableVersions(String libName) {
- Document doc = null;
+ Document doc;
  List<String> versions = new ArrayList<>();
  try{
- doc = Jsoup.connect("https://repo1.maven.org/maven2/org/apache/jmeter/" + libName).get();
- Elements links = doc.select("a[href]");
+ doc = Jsoup.connect(mavenRepoUrl + libName).get();
+ Elements links = doc.select("a[title]");
  for(Element link : links) {
  String href = link.attr("href");
- if(href.contains(libName + "-")) {
- String version = href.substring(href.lastIndexOf("-") + 1, href.lastIndexOf(".jar"));
+ if(!href.contains("xml")){
+ String version = mavenRepoUrl + libName + "/" + href + libName + "-" + href.substring(0, href.lastIndexOf('/')) + ".jar";
  versions.add(version);
  }
  }
@@ -394,8 +398,7 @@ public void updateCustomPluginInfoInDB(JSONObject pluginModel) {
  if(conn == null || conn.isClosed()){
  conn = SQLiteConnectionPool.getConnection();
  }
- //"CREATE TABLE plugins (id TEXT PRIMARY KEY, name TEXT, type TEXT, description TEXT, helpUrl TEXT, markerClass TEXT, screenshotUrl TEXT, vendor TEXT, componentClasses TEXT, versions_count INTEGER)";
- //id, name, type, description, helpUrl, markerClass, screenshotUrl, vendor, componentClasses, versions_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
  PreparedStatement preparedStatement = conn.prepareStatement(INSERT_PLUGIN_INFO);
  preparedStatement.setString(1, pluginModel.getString("id"));
  preparedStatement.setString(2, pluginModel.getString("name"));
@@ -405,7 +408,7 @@ public void updateCustomPluginInfoInDB(JSONObject pluginModel) {
  preparedStatement.setString(6, pluginModel.getString("markerClass"));
  preparedStatement.setString(7, pluginModel.getString("screenshotUrl"));
  preparedStatement.setString(8, pluginModel.getString("vendor"));
- preparedStatement.setString(9, pluginModel.getString("componentClasses"));
+ preparedStatement.setString(9, pluginModel.has("componentClasses") ? pluginModel.getString("componentClasses") : null);
  preparedStatement.setDouble(10, pluginModel.getDouble("version_count"));
 
  int rowsInserted = preparedStatement.executeUpdate();

src/main/java/io/perfwise/local/pluginsmanager/scheduler/parser/Parse.java

@@ -62,7 +62,7 @@ public static void downloadAllPlugins(JSONArray jmeterRepoJson) throws URISyntax
  LOGGER.info("Downloaded all the available plugins from the market");
  }
 
- public static int getLocalPluginCount(JSONArray pluginsArray) throws SQLException, InterruptedException {
+ public static int getLocalPluginCount() throws SQLException, InterruptedException {
  int localStoreCount = 0;
  conn = SQLiteConnectionPool.getConnection();
 
@@ -80,7 +80,7 @@ public static int getLocalPluginCount(JSONArray pluginsArray) throws SQLExceptio
  return localStoreCount;
  }
 
- public static void addPluginDataToDB(JSONObject pluginObj) throws SQLException, InterruptedException {
+ public static void addPluginDataToDB(JSONObject pluginObj) {
  String type = "custom";
  httpRequest.updateCustomPluginInfoInDB(pluginObj);
  }

src/main/resources/config.properties

@@ -11,7 +11,7 @@ db.timeout.secs=300
 
 #External APIs
 jmeter.plugins.url=https://jmeter-plugins.org/repo/
-mvn.repo.url=https://mvnrepository.com/search?q=
+mvn.repo.url=https://repo1.maven.org/maven2/org/apache/jmeter/
 
 #Directory Configs
 #local.repo.path=/app/plugins-manager/