Merge branch 'master' of https://github.com/shruthikmusukula/NavalDat…

…aAnalyticsPlatform Merge
shruthikmusukula · Aug 3, 2020 · dd476f9 · dd476f9
2 parents 491e785 + f414bce
commit dd476f9
Show file tree

Hide file tree

Showing 15 changed files with 393 additions and 0 deletions.
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 The Naval Postgraduate School
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/webapp/WEB-INF/classes/1 b/webapp/WEB-INF/classes/1
@@ -0,0 +1,13 @@
+<!DOCTYPE HTML><html lang="en"><head>
+<meta charset="UTF-8">
+<title>DiDAP Job Monitor</title>
+</head>
+<body>
+<p>
+<h2>DiDAP Job Monitor</h2>
+<ul>
+<li><a href="HelloWorldExample">Hello World Servlet</a></li>
+<li><a href="MSTIFF">Compile and Run MSTIFF Extraction Code</a></li>
+</ul>
+</body></html>
+
diff --git a/webapp/WEB-INF/classes/HelloWorld.class b/webapp/WEB-INF/classes/HelloWorld.class
diff --git a/webapp/WEB-INF/classes/HelloWorld.java b/webapp/WEB-INF/classes/HelloWorld.java
@@ -0,0 +1,40 @@
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ResourceBundle;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+public class HelloWorld extends HttpServlet {
+
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public void doGet(HttpServletRequest request,
+                      HttpServletResponse response)
+        throws IOException, ServletException
+    {
+        response.setContentType("text/html");
+        response.setCharacterEncoding("UTF-8");
+        PrintWriter out = response.getWriter();
+
+        out.println("<!DOCTYPE html><html>");
+        out.println("<head>");
+        out.println("<meta charset=\"UTF-8\" />");
+
+	String title = "Hello, World!";
+
+        out.println("<title>" + title + "</title>");
+        out.println("</head>");
+        out.println("<body bgcolor=\"white\">");
+
+        out.println("<h1>" + title + "</h1>");
+        out.println("</body>");
+        out.println("</html>");
+    }
+}
+
+
+
diff --git a/webapp/WEB-INF/classes/MetadataTableCreation.class b/webapp/WEB-INF/classes/MetadataTableCreation.class
diff --git a/webapp/WEB-INF/classes/MetadataTableCreation.java b/webapp/WEB-INF/classes/MetadataTableCreation.java
@@ -0,0 +1,136 @@
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.ResourceBundle;
+import java.util.Arrays;
+
+import java.sql.SQLException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.sql.DriverManager;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+public class MetadataTableCreation extends HttpServlet {
+	private Connection conn;
+	private Statement stmt;
+	private boolean connEstablished = false;
+	private PrintWriter out;
+
+	public void DBinit() {
+		if (!connEstablished) {
+			try {
+				// Register driver and create driver instance
+				String driverName = "com.cloudera.hive.jdbc4.HS1Driver";
+				Class.forName(driverName);
+			} catch (ClassNotFoundException e) {
+				e.printStackTrace(out);
+				System.exit(1);
+			}
+
+			try {
+				// Establish connection
+				String url = "jdbc:hive://localhost:10000/metadata";
+				conn = DriverManager.getConnection(url, "", "");
+				out.println("Connected to Database!");
+
+				// Create statement
+				stmt = conn.createStatement();
+				connEstablished = true;
+			} catch (SQLException e) {
+				e.printStackTrace(out);
+				System.exit(1);
+			}
+		}
+	}
+
+	public void createTable(String tableName) {
+		if (connEstablished) {
+			try {
+				String query = "";
+				// create hive table statement
+				query = "CREATE EXTERNAL TABLE IF NOT EXISTS " + tableName
+			+ " (filePath STRING, dateStamp INT,navTime STRING,FathTime STRING,latitude DOUBLE,longitude DOUBLE,waterDepth FLOAT,towfishDepth FLOAT)"
+			+ "ROW FORMAT DELIMITED"
+			+ "FIELDS TERMINATED BY ','"
+			+ "STORED AS TEXTFILE"
+			+ "LOCATION '/user/cloudera/successfulOutput_fullScript'";
+				out.println("Your query:" + query);
+				stmt.executeQuery(query); // executeUpdate
+			} catch (SQLException e) {
+				e.printStackTrace();
+				out.println("Query Failed!!");
+			}
+		}
+	}
+
+	public void DBExecuteQuery(String query) {
+		if (connEstablished) {
+			try {
+				out.println("Running: " + query);
+				stmt.execute(query);
+				out.println("Finished!");
+			} catch (SQLException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	public void DBfinish() {
+		if (connEstablished) {
+			out.println("Connection Closed.");
+			try {
+				stmt.close();
+				conn.close();
+				connEstablished = false;
+			} catch (SQLException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+    @Override
+    public void doGet(HttpServletRequest request,
+                      HttpServletResponse response)
+        throws IOException, ServletException
+    {
+        response.setContentType("text/html");
+        response.setCharacterEncoding("UTF-8");
+        out = response.getWriter();
+
+        out.println("<!DOCTYPE html><html>");
+        out.println("<head>");
+        out.println("<meta charset=\"UTF-8\" />");
+
+        String title = "Metadata Table Creation";
+
+        out.println("<title>" + title + "</title>");
+        out.println("</head>");
+
+        out.println("<body bgcolor=\"white\">");
+
+        out.println("<h1>" + title + " Status:</h1>\n");
+		out.println("Metadata Table Data Being Loaded from HDFS:");
+
+		// Create DB Connection
+        DBinit();
+
+		// Create Actual Table
+		//createTable("florida");
+
+		// Display HDFS Data
+        String query = "LOAD DATA INPATH '/user/cloudera/successfulOutput_fullScript/part-m-00000' OVERWRITE INTO TABLE florida";
+		DBExecuteQuery(query);
+
+        out.println("</body>");
+        out.println("</html>");
+
+        // Close DB Connection
+        DBfinish();
+    }
+}
diff --git a/webapp/WEB-INF/classes/MstiffExtraction.class b/webapp/WEB-INF/classes/MstiffExtraction.class
diff --git a/webapp/WEB-INF/classes/MstiffExtraction.java b/webapp/WEB-INF/classes/MstiffExtraction.java
@@ -0,0 +1,58 @@
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.ResourceBundle;
+import java.util.Arrays;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+public class MstiffExtraction extends HttpServlet {
+
+    @Override
+    public void doGet(HttpServletRequest request,
+                      HttpServletResponse response)
+        throws IOException, ServletException
+    {
+        response.setContentType("text/html");
+        response.setCharacterEncoding("UTF-8");
+        PrintWriter out = response.getWriter();
+
+        out.println("<!DOCTYPE html><html>");
+        out.println("<head>");
+        out.println("<meta charset=\"UTF-8\" />");
+
+        String title = "MSTIFF Extraction";
+
+        out.println("<title>" + title + "</title>");
+        out.println("</head>");
+        out.println("<body bgcolor=\"white\">");
+
+        out.println("<h1>" + title + " Job Status:</h1>\n");
+
+		String[] command = {"/bin/bash", "/usr/share/tomcat/webapps/didap/WEB-INF/scripts/mstiff.sh"};
+        ProcessBuilder p = new ProcessBuilder(command);
+        Process p2 = p.start();
+        try {
+            p2.waitFor();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+			System.out.println("Did not execute in hadoop!");
+        }
+
+        BufferedReader br = new BufferedReader(new InputStreamReader(p2.getInputStream()));
+        String line;
+
+		out.println("Output of running " + Arrays.toString(command) + " is: ");
+        while ((line = br.readLine()) != null) {
+            out.println(line + "\n");
+        }
+
+        out.println("</body>");
+        out.println("</html>");
+    }
+}
+
diff --git a/webapp/WEB-INF/scripts/1 b/webapp/WEB-INF/scripts/1
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+su cloudera <<!
+cloudera
+
+javac -cp /usr/lib/hadoop/*:/usr/lib/hadoop/client-0.20/* -d /home/cloudera/MSTIFF_Code /home/cloudera/MSTIFF_Code/GET_MST_mod.java
+jar -cvf GET_MST_mod.jar -C /home/cloudera/MSTIFF_Code/ .
+
+# move JAR File to specified directory on the vm
+mv GET_MST_mod.jar /home/cloudera/MSTIFF_Code
+
+#su cloudera <<!
+#cloudera
+
+#chown cloudera:cloudera /home/cloudera/MSTIFF_Code/GET_MST_MOD.jar
+#echo $?
+#chmod ug+rw /home/cloudera/MSTIFF_Code/GET_MST_MOD.jar
+#echo $?
+#!
+
+hadoop jar /home/cloudera/MSTIFF_Code/GET_MST_mod.jar GET_MST_mod /user/cloudera/MSTIFF_Code /user/cloudera/successfulOutput1
+echo $?
+#!
diff --git a/webapp/WEB-INF/scripts/mstiff.sh b/webapp/WEB-INF/scripts/mstiff.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+javac -cp /usr/lib/hadoop/*:/usr/lib/hadoop/client-0.20/* -d /home/cloudera/MSTIFF_Code /home/cloudera/MSTIFF_Code/GET_MST_mod.java & jar -cvf GET_MST_mod.jar -C /home/cloudera/MSTIFF_Code/ .
+
+hadoop jar /home/cloudera/GET_MST_mod.jar GET_MST_mod /user/cloudera/MSTIFF_Code /user/cloudera/successfulOutput_fullScript 
+
diff --git a/webapp/WEB-INF/web.xml b/webapp/WEB-INF/web.xml
@@ -0,0 +1,26 @@
+<web-app>
+  <servlet>
+    <servlet-name>HelloWorld</servlet-name>
+    <servlet-class>HelloWorld</servlet-class>
+  </servlet>
+  <servlet>
+    <servlet-name>MetadataTableCreation</servlet-name>
+    <servlet-class>MetadataTableCreation</servlet-class>
+  </servlet>
+  <servlet>
+    <servlet-name>MSTIFF_Extraction</servlet-name>
+    <servlet-class>MstiffExtraction</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>HelloWorld</servlet-name>
+    <url-pattern>/HelloWorld</url-pattern>
+  </servlet-mapping>
+  <servlet-mapping>
+    <servlet-name>MetadataTableCreation</servlet-name>
+    <url-pattern>/MetadataTableCreation</url-pattern>
+  </servlet-mapping>
+  <servlet-mapping>
+    <servlet-name>MSTIFF_Extraction</servlet-name>
+    <url-pattern>/MSTIFF_Extraction</url-pattern>
+  </servlet-mapping>
+</web-app>
diff --git a/webapp/WEB-INF/web.xml~ b/webapp/WEB-INF/web.xml~
@@ -0,0 +1,26 @@
+<web-app>
+  <servlet>
+    <servlet-name>HelloWorld</servlet-name>
+    <servlet-class>HelloWorld</servlet-class>
+  </servlet>
+  <servlet>
+    <servlet-name>MetadataTableCreation</servlet-name>
+    <servlet-class>MetadataTableCreation</servlet-class>
+  </servlet>
+  <servlet>
+    <servlet-name>MSTIFF_Extraction</servlet-name>
+    <servlet-class>MstiffExtraction</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>HelloWorld</servlet-name>
+    <url-pattern>/HelloWorld</url-pattern>
+  </servlet-mapping>
+  <servlet-mapping>
+    <servlet-name>MetadataTableCreation</servlet-name>
+    <url-pattern>/MetadataTableCreation</url-pattern>
+  </servlet-mapping>
+  <servlet-mapping>
+    <servlet-name>MSTIFF_Extraction</servlet-name>
+    <url-pattern>/MSTIFF_Extraction</url-pattern>
+  </servlet-mapping>
+</web-app>
diff --git a/webapp/WEB-INF/web_my.xml b/webapp/WEB-INF/web_my.xml
@@ -0,0 +1,18 @@
+<web-app>
+  <servlet>
+    <servlet-name>HelloWorld</servlet-name>
+    <servlet-class>HelloWorld</servlet-class>
+  </servlet>
+  <servlet>
+    <servlet-name>MSTIFF_Extraction</servlet-name>
+    <servlet-class>MstiffExtraction</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>HelloWorld</servlet-name>
+    <url-pattern>/HelloWorld</url-pattern>
+  </servlet-mapping>
+  <servlet-mapping>
+    <servlet-name>MSTIFF_Extraction</servlet-name>
+    <url-pattern>/MSTIFF_Extraction</url-pattern>
+  </servlet-mapping>
+</web-app>
diff --git a/webapp/index.html b/webapp/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE HTML><html lang="en"><head>
+<meta charset="UTF-8">
+<title>DiDAP Job Monitor</title>
+</head>
+<body>
+<p>
+<h2>DiDAP Job Monitor</h2>
+<ul>
+<li><a href="MetadataTableCreation">Moving Extracted Data to Hive</a></li>
+<li><a href="MSTIFF_Extraction">Compile and Run MSTIFF Extraction Code</a></li>
+</ul>
+</body></html>
+