Merge pull request #107 from sethsec/dev

Version 1.2-beta

lib/config_parser.py

@@ -1,4 +1,3 @@
-import simplejson
 from collections import OrderedDict
 from ConfigParser import ConfigParser
 
@@ -23,6 +22,28 @@ def read_config_ini(config_file=None):
             supported_services.append(service)
     return config,supported_services
 
+def get_concurrent_tasks(config_file):
+    config,supported_services = read_config_ini(config_file)
+    for (key, val) in config.items("celerystalk-config"):
+        if key == "concurrent_tasks":
+            return val
+
+def get_simpleserver_port(config_file):
+    config,supported_services = read_config_ini(config_file)
+    for (key, val) in config.items("celerystalk-config"):
+        if key == "simple_server_port":
+            return val
+
+def get_screenshot_max(config_file):
+    config,supported_services = read_config_ini(config_file)
+    for (key, val) in config.items("celerystalk-config"):
+        if key == "max_screenshots_per_vhost":
+            return val
+
+def get_user_config(config_file):
+    config,supported_services = read_config_ini(config_file)
+    return config.items("user-config")
+
 
 def extract_bb_nmap_options(config_file=None):
     config = ConfigParser(allow_no_value=True)
@@ -44,8 +65,6 @@ def extract_udp_scan_nmap_options(config_file=None):
             options = udp_nmap_command.replace('nmap', '').replace('[TARGET]', '')
             return options
 
-
-
 def read_bb_scope_ini(bb_scope_file):
     bb_config = ConfigParser(allow_no_value=True)
     bb_config.read([bb_scope_file])

lib/csimport.py

@@ -237,7 +237,7 @@ def import_url(url,workspace,output_base_dir):
                     url_path = ''
 
                 url_screenshot_filename = scan_output_base_file_dir + url_path.replace("/", "_") + ".png"
-                db_path = (vhost, port, url, 0, url_screenshot_filename, workspace)
+                db_path = (vhost, port, url.rstrip("/"), 0, url_screenshot_filename, workspace)
                 db.insert_new_path(db_path)
                 # print("Found Url: " + str(url))
                 #urls_to_screenshot.append((url, url_screenshot_filename))
@@ -246,7 +246,7 @@ def import_url(url,workspace,output_base_dir):
                 # print(result)
 
 
-            db_path = (vhost, port, url, 0, url_screenshot_filename, workspace)
+            db_path = (vhost, port, url.rstrip("/"), 0, url_screenshot_filename, workspace)
             lib.db.insert_new_path(db_path)
     else:
         print("[!] {0} is explicitly marked as out of scope. Skipping...".format(vhost))
@@ -430,9 +430,9 @@ def process_nmap_data(nmap_report,workspace, target=None):
         unique_db_ips = lib.db.is_vhost_in_db(ip,workspace) #Returns data if IP is in database
         #print(unique_db_ips)
         vhosts = scanned_host.hostnames
-        print("process_nmap_data: " + str(vhosts))
+        #print("process_nmap_data: " + str(vhosts))
         for vhost in vhosts:
-            print("process_nmap_data: " + vhost)
+            #print("process_nmap_data: " + vhost)
             vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(vhost, workspace)
             if not vhost_explicitly_out_of_scope:  # if the vhost is not explicitly out of scope, add it to db
                 is_vhost_in_db = lib.db.is_vhost_in_db(vhost, workspace)  # Returns data if IP is in database
@@ -522,7 +522,8 @@ def process_nmap_data(nmap_report,workspace, target=None):
                 scan_output_base_file_dir = os.path.abspath(output_base_dir + file_end_part)
 
                 if (scanned_service_name == 'https') or (scanned_service_name == 'http'):
-                    path = scanned_service_name + "://" + ip + ":" + str(scanned_service_port) + "/"
+                    path = scanned_service_name + "://" + ip + ":" + str(scanned_service_port)
+                    path = path.rstrip("/")
                     db_path = db.get_path(path, workspace)
                     if not db_path:
                         url_screenshot_filename = scan_output_base_file_dir + ".png"
@@ -531,7 +532,7 @@ def process_nmap_data(nmap_report,workspace, target=None):
 
 
                 for vhost in vhosts:
-                    print("process_nmap_data - add service: " + vhost)
+                    #print("process_nmap_data - add service: " + vhost)
                     db_service = db.get_service(vhost, scanned_service_port, scanned_service_protocol, workspace)
                     if not db_service:
                         print("service didnt exist, adding: " + vhost + str(scanned_service_port))
@@ -552,7 +553,8 @@ def process_nmap_data(nmap_report,workspace, target=None):
                             scanned_service_port) + "_" + scanned_service_protocol)
 
                     if (scanned_service_name == 'https') or (scanned_service_name == 'http'):
-                        path = scanned_service_name + "://" + vhost + ":" + str(scanned_service_port) + "/"
+                        path = scanned_service_name + "://" + vhost + ":" + str(scanned_service_port)
+                        path = path.rstrip("/")
                         db_path = db.get_path(path, workspace)
                         if not db_path:
                             url_screenshot_filename = scan_output_base_file_dir + ".png"

lib/db.py

@@ -662,6 +662,12 @@ def get_all_paths_for_host_path_only(ip,workspace):
     CONNECTION.commit()
     return all_paths_for_host
 
+def get_x_paths_for_host_path_only(ip,workspace,config_max):
+    CUR.execute("SELECT path FROM paths WHERE ip = ? AND workspace = ? LIMIT ?", (ip,workspace,config_max))
+    all_paths_for_host = CUR.fetchall()
+    CONNECTION.commit()
+    return all_paths_for_host
+
 def get_path(path,workspace):
     CUR.execute("SELECT * FROM paths WHERE workspace = ? AND path = ?", (workspace,path))
     path = CUR.fetchall()
@@ -677,3 +683,7 @@ def get_unique_hosts_with_paths(workspace):
 def update_path(path,submitted,workspace):
     CUR.execute("UPDATE paths SET submitted=? WHERE path=? AND workspace=?", (submitted,path,workspace))
     CONNECTION.commit()
+
+def update_path_with_filename(path,filename,workspace):
+    CUR.execute("UPDATE paths SET url_screenshot_filename=? WHERE path=? AND workspace=?", (filename,path,workspace))
+    CONNECTION.commit()

lib/nmap.py

@@ -43,7 +43,7 @@ def nmap_scan_subdomain_host(vhost,workspace,simulation,output_base_dir,config_f
         )()
 
 def nmapcommand(simulation,targets,config_file=None):
-    lib.utils.start_services()
+    lib.utils.start_services(config_file)
     task_count = 0
 
     try:

lib/query.py

@@ -75,8 +75,10 @@ def query_sqlite(workspace, target=None, repeat=None, summary=None):
 
             command_length = len(command)
             if command_length > terminal_width - 11:
-
-                print("  [" + str(id) + "]\t" + command[0:terminal_width - 11] + "...")
+                if int(id) > 999:
+                    print(" [" + str(id) + "]\t" + command[0:terminal_width - 11] + "...")
+                else:
+                    print("  [" + str(id) + "]\t" + command[0:terminal_width - 11] + "...")
             else:
                 print("  [" + str(id) + "]\t" + command)
         if repeat:
@@ -152,7 +154,10 @@ def query_sqlite(workspace, target=None, repeat=None, summary=None):
                 id_str = str(id) + " "
             command_length = len(command)
             if command_length > terminal_width -22:
-                print("  [" + id_str + "][" + run_time + "] " + command[0:terminal_width - 22] + "...")
+                if int(id) > 999:
+                    print(" [" + id_str + "][" + run_time + "] " + command[0:terminal_width - 22] + "...")
+                else:
+                    print("  [" + id_str + "][" + run_time + "] " + command[0:terminal_width - 22] + "...")
             else:
                 print("  [" + id_str + "][" + run_time + "] " + command)
     else: