diff --git a/.gitignore b/.gitignore index 519328f..f2fcbcb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__ *.zip .idea venv +lock.pid diff --git a/README.md b/README.md index 1e95715..461fe1c 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ In order to use the multicast distribution for files, please make sure that the The list contains all Hashcat versions with which the client was tested and is able to work with (other versions might work): +* 6.0.0 +* 5.1.0 * 5.0.0 * 4.2.1 * 4.2.0 diff --git a/__main__.py b/__main__.py index 045ede7..987741f 100644 --- a/__main__.py +++ b/__main__.py @@ -175,9 +175,14 @@ def loop(): task_change = True task.reset_task() continue - # if prince is used, make sure it's downloaded - if task.get_task()['usePrince']: - binaryDownload.check_prince() + # if prince is used, make sure it's downloaded (deprecated, as preprocessors are integrated generally now) + if 'usePrince' in task.get_task() and task.get_task()['usePrince']: + if not binaryDownload.check_prince(): + continue + # if preprocessor is used, make sure it's downloaded + if 'usePreprocessor' in task.get_task() and task.get_task()['usePreprocessor']: + if not binaryDownload.check_preprocessor(task): + continue # check if all required files are present if not files.check_files(task.get_task()['files'], task.get_task()['taskId']): task.reset_task() @@ -204,7 +209,7 @@ def loop(): continue elif chunk_resp == -1: # measure keyspace - if not cracker.measure_keyspace(task.get_task(), chunk): # failure case + if not cracker.measure_keyspace(task, chunk): # failure case task.reset_task() continue elif chunk_resp == -3: @@ -249,7 +254,7 @@ def loop(): # run chunk logging.info("Start chunk...") - cracker.run_chunk(task.get_task(), chunk.chunk_data()) + cracker.run_chunk(task.get_task(), chunk.chunk_data(), task.get_preprocessor()) if cracker.agent_stopped(): # if the chunk was aborted by a stop from the server, we need to ask for a task again first task.reset_task() diff --git a/changelog.md b/changelog.md index 5193e02..c9ea568 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,14 @@ +## v0.5.0 -> v0.6.0 + +### Features + +* Generic integration of preprocessors. +* Added compatibility for newer Hashcat versions with different outfile format specification. + +### Bugfixes + +* Fixed crash handling on generic cracker for invalid binary names + ## v0.4.0 -> v0.5.0 ### Enhancements diff --git a/htpclient/binarydownload.py b/htpclient/binarydownload.py index 6272aa3..9477432 100644 --- a/htpclient/binarydownload.py +++ b/htpclient/binarydownload.py @@ -132,6 +132,45 @@ def check_prince(self): os.rmdir("temp") logging.debug("PRINCE downloaded and extracted") return True + + def check_preprocessor(self, task): + logging.debug("Checking if requested preprocessor is present...") + path = "preprocessor/" + str(task.get_task()['preprocessor']) + "/" + query = copy_and_set_token(dict_downloadBinary, self.config.get_value('token')) + query['type'] = 'preprocessor' + query['preprocessorId'] = task.get_task()['preprocessor'] + req = JsonRequest(query) + ans = req.execute() + if ans is None: + logging.error("Failed to load preprocessor settings!") + sleep(5) + return False + elif ans['response'] != 'SUCCESS' or not ans['url']: + logging.error("Getting preprocessor settings failed: " + str(ans)) + sleep(5) + return False + else: + task.set_preprocessor(ans) + if os.path.isdir(path): # if it already exists, we don't need to download it + logging.debug("Preprocessor is already downloaded") + return True + logging.debug("Preprocessor not found, download...") + if not Download.download(ans['url'], "temp.7z"): + logging.error("Download of preprocessor failed!") + sleep(5) + return False + if Initialize.get_os() == 1: + os.system("7zr" + Initialize.get_os_extension() + " x -otemp temp.7z") + else: + os.system("./7zr" + Initialize.get_os_extension() + " x -otemp temp.7z") + for name in os.listdir("temp"): # this part needs to be done because it is compressed with the main subfolder of prince + if os.path.isdir("temp/" + name): + os.rename("temp/" + name, path) + break + os.unlink("temp.7z") + os.rmdir("temp") + logging.debug("Preprocessor downloaded and extracted") + return True def check_version(self, cracker_id): path = "crackers/" + str(cracker_id) + "/" diff --git a/htpclient/generic_cracker.py b/htpclient/generic_cracker.py index 0859525..b4536f7 100644 --- a/htpclient/generic_cracker.py +++ b/htpclient/generic_cracker.py @@ -1,173 +1,179 @@ -import logging -import subprocess -from queue import Queue, Empty -from threading import Thread - -from htpclient.config import Config -from htpclient.generic_status import GenericStatus -from htpclient.initialize import Initialize -from htpclient.jsonRequest import JsonRequest -from htpclient.dicts import * - - -class GenericCracker: - def __init__(self, cracker_id, binary_download): - self.config = Config() - self.io_q = Queue() - self.callPath = "../crackers/" + str(cracker_id) + "/" + binary_download.get_version()['executable'] - self.executable_name = binary_download.get_version()['executable'] - self.keyspace = 0 - - def run_chunk(self, task, chunk): - args = " crack -s " + str(chunk['skip']) - args += " -l " + str(chunk['length']) - args += " " + task['attackcmd'].replace(task['hashlistAlias'], "../hashlists/" + str(task['hashlistId'])) - full_cmd = self.callPath + args - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - logging.debug("CALL: " + full_cmd) - process = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd='files') - - logging.debug("started cracking") - out_thread = Thread(target=self.stream_watcher, name='stdout-watcher', args=('OUT', process.stdout)) - err_thread = Thread(target=self.stream_watcher, name='stderr-watcher', args=('ERR', process.stderr)) - out_thread.start() - err_thread.start() - - main_thread = Thread(target=self.run_loop, name='run_loop', args=(process, chunk, task)) - main_thread.start() - - # wait for all threads to finish - process.wait() - out_thread.join() - err_thread.join() - logging.info("finished chunk") - - def run_loop(self, process, chunk, task): - cracks = [] - while True: - try: - # Block for 1 second. - item = self.io_q.get(True, 1) - except Empty: - # No output in either streams for a second. Are we done? - if process.poll() is not None: - # is the case when the process is finished - break - else: - identifier, line = item - if identifier == 'OUT': - status = GenericStatus(line.decode()) - if status.is_valid(): - # send update to server - progress = status.get_progress() - speed = status.get_speed() - initial = True - while cracks or initial: - initial = False - cracks_backup = [] - if len(cracks) > 1000: - # we split - cnt = 0 - new_cracks = [] - for crack in cracks: - cnt += 1 - if cnt > 1000: - cracks_backup.append(crack) - else: - new_cracks.append(crack) - cracks = new_cracks - - query = copy_and_set_token(dict_sendProgress, self.config.get_value('token')) - query['chunkId'] = chunk['chunkId'] - query['keyspaceProgress'] = chunk['skip'] - query['relativeProgress'] = progress - query['speed'] = speed - query['state'] = (4 if progress == 10000 else 2) - query['cracks'] = cracks - req = JsonRequest(query) - - logging.debug("Sending " + str(len(cracks)) + " cracks...") - ans = req.execute() - if ans is None: - logging.error("Failed to send solve!") - elif ans['response'] != 'SUCCESS': - logging.error("Error from server on solve: " + str(ans)) - else: - if ans['zaps']: - with open("files/zap", "wb") as zapfile: # need to check if we are in the main dir here - zapfile.write('\n'.join(ans['zaps']).encode()) - zapfile.close() - cracks = cracks_backup - logging.info( - "Progress: " + str(progress / 100) + "% Cracks: " + str(len(cracks)) + - " Accepted: " + str(ans['cracked']) + " Skips: " + str(ans['skipped']) + " Zaps: " + str(len(ans['zaps']))) - else: - line = line.decode() - if ":" in line: - cracks.append(line.strip()) - else: - logging.warning("OUT: " + line.strip()) - else: - print("ERROR: " + str(line).strip()) - # TODO: send error and abort cracking - - def measure_keyspace(self, task, chunk): - full_cmd = self.callPath + " keyspace " + task['attackcmd'].replace("-a " + task['hashlistAlias'] + " ", "") - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - output = subprocess.check_output(full_cmd, shell=True, cwd='files') - output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") - keyspace = "0" - for line in output: - if not line: - continue - keyspace = line - self.keyspace = int(keyspace) - return chunk.send_keyspace(int(keyspace), task['taskId']) - - def run_benchmark(self, task): - ksp = self.keyspace - if ksp == 0: - ksp = task['keyspace'] - args = task['attackcmd'].replace(task['hashlistAlias'], "../hashlists/" + str(task['hashlistId'])) - full_cmd = self.callPath + " crack " + args + " -s 0 -l " + str(ksp) + " --timeout=" + str(task['bench']) - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - logging.debug("CALL: " + full_cmd) - output = subprocess.check_output(full_cmd, shell=True, cwd='files') - if output: - output = output.replace(b"\r\n", b"\n").decode('utf-8') - output = output.split('\n') - last_valid_status = None - for line in output: - if not line: - continue - status = GenericStatus(line) - if status.is_valid(): - last_valid_status = status - if last_valid_status is None: - query = copy_and_set_token(dict_clientError, self.config.get_value('token')) - query['taskId'] = task['taskId'] - query['message'] = "Generic benchmark failed!" - req = JsonRequest(query) - req.execute() - return 0 - return float(last_valid_status.get_progress()) / 10000 - else: - query = copy_and_set_token(dict_clientError, self.config.get_value('token')) - query['taskId'] = task['taskId'] - query['message'] = "Generic benchmark gave no output!" - req = JsonRequest(query) - req.execute() - return 0 - - def stream_watcher(self, identifier, stream): - for line in stream: - self.io_q.put((identifier, line)) - - if not stream.closed: - stream.close() - - def agent_stopped(self): - return False +import logging +import subprocess +from queue import Queue, Empty +from threading import Thread + +from htpclient.config import Config +from htpclient.generic_status import GenericStatus +from htpclient.helpers import send_error +from htpclient.initialize import Initialize +from htpclient.jsonRequest import JsonRequest +from htpclient.dicts import * + + +class GenericCracker: + def __init__(self, cracker_id, binary_download): + self.config = Config() + self.io_q = Queue() + self.callPath = "../crackers/" + str(cracker_id) + "/" + binary_download.get_version()['executable'] + self.executable_name = binary_download.get_version()['executable'] + self.keyspace = 0 + + def run_chunk(self, task, chunk): + args = " crack -s " + str(chunk['skip']) + args += " -l " + str(chunk['length']) + args += " " + task['attackcmd'].replace(task['hashlistAlias'], "../hashlists/" + str(task['hashlistId'])) + full_cmd = self.callPath + args + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + logging.debug("CALL: " + full_cmd) + process = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd='files') + + logging.debug("started cracking") + out_thread = Thread(target=self.stream_watcher, name='stdout-watcher', args=('OUT', process.stdout)) + err_thread = Thread(target=self.stream_watcher, name='stderr-watcher', args=('ERR', process.stderr)) + out_thread.start() + err_thread.start() + + main_thread = Thread(target=self.run_loop, name='run_loop', args=(process, chunk, task)) + main_thread.start() + + # wait for all threads to finish + process.wait() + out_thread.join() + err_thread.join() + logging.info("finished chunk") + + def run_loop(self, process, chunk, task): + cracks = [] + while True: + try: + # Block for 1 second. + item = self.io_q.get(True, 1) + except Empty: + # No output in either streams for a second. Are we done? + if process.poll() is not None: + # is the case when the process is finished + break + else: + identifier, line = item + if identifier == 'OUT': + status = GenericStatus(line.decode()) + if status.is_valid(): + # send update to server + progress = status.get_progress() + speed = status.get_speed() + initial = True + while cracks or initial: + initial = False + cracks_backup = [] + if len(cracks) > 1000: + # we split + cnt = 0 + new_cracks = [] + for crack in cracks: + cnt += 1 + if cnt > 1000: + cracks_backup.append(crack) + else: + new_cracks.append(crack) + cracks = new_cracks + + query = copy_and_set_token(dict_sendProgress, self.config.get_value('token')) + query['chunkId'] = chunk['chunkId'] + query['keyspaceProgress'] = chunk['skip'] + query['relativeProgress'] = progress + query['speed'] = speed + query['state'] = (4 if progress == 10000 else 2) + query['cracks'] = cracks + req = JsonRequest(query) + + logging.debug("Sending " + str(len(cracks)) + " cracks...") + ans = req.execute() + if ans is None: + logging.error("Failed to send solve!") + elif ans['response'] != 'SUCCESS': + logging.error("Error from server on solve: " + str(ans)) + else: + if ans['zaps']: + with open("files/zap", "wb") as zapfile: # need to check if we are in the main dir here + zapfile.write('\n'.join(ans['zaps']).encode()) + zapfile.close() + cracks = cracks_backup + logging.info( + "Progress: " + str(progress / 100) + "% Cracks: " + str(len(cracks)) + + " Accepted: " + str(ans['cracked']) + " Skips: " + str(ans['skipped']) + " Zaps: " + str(len(ans['zaps']))) + else: + line = line.decode() + if ":" in line: + cracks.append(line.strip()) + else: + logging.warning("OUT: " + line.strip()) + else: + print("ERROR: " + str(line).strip()) + # TODO: send error and abort cracking + + def measure_keyspace(self, task, chunk): + full_cmd = self.callPath + " keyspace " + task['attackcmd'].replace("-a " + task['hashlistAlias'] + " ", "") + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + try: + output = subprocess.check_output(full_cmd, shell=True, cwd='files') + except subprocess.CalledProcessError as e: + logging.error("Error during keyspace measurement: " + str(e)) + send_error(str(e), self.config.get_value('token'), task['taskId'], chunk.chunk_data()['chunkId']) + return False + output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") + keyspace = "0" + for line in output: + if not line: + continue + keyspace = line + self.keyspace = int(keyspace) + return chunk.send_keyspace(int(keyspace), task['taskId']) + + def run_benchmark(self, task): + ksp = self.keyspace + if ksp == 0: + ksp = task['keyspace'] + args = task['attackcmd'].replace(task['hashlistAlias'], "../hashlists/" + str(task['hashlistId'])) + full_cmd = self.callPath + " crack " + args + " -s 0 -l " + str(ksp) + " --timeout=" + str(task['bench']) + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + logging.debug("CALL: " + full_cmd) + output = subprocess.check_output(full_cmd, shell=True, cwd='files') + if output: + output = output.replace(b"\r\n", b"\n").decode('utf-8') + output = output.split('\n') + last_valid_status = None + for line in output: + if not line: + continue + status = GenericStatus(line) + if status.is_valid(): + last_valid_status = status + if last_valid_status is None: + query = copy_and_set_token(dict_clientError, self.config.get_value('token')) + query['taskId'] = task['taskId'] + query['message'] = "Generic benchmark failed!" + req = JsonRequest(query) + req.execute() + return 0 + return float(last_valid_status.get_progress()) / 10000 + else: + query = copy_and_set_token(dict_clientError, self.config.get_value('token')) + query['taskId'] = task['taskId'] + query['message'] = "Generic benchmark gave no output!" + req = JsonRequest(query) + req.execute() + return 0 + + def stream_watcher(self, identifier, stream): + for line in stream: + self.io_q.put((identifier, line)) + + if not stream.closed: + stream.close() + + def agent_stopped(self): + return False diff --git a/htpclient/hashcat_cracker.py b/htpclient/hashcat_cracker.py index 6efae14..fa9331b 100644 --- a/htpclient/hashcat_cracker.py +++ b/htpclient/hashcat_cracker.py @@ -1,508 +1,611 @@ -import string -import logging -import subprocess -from time import sleep -from queue import Queue, Empty -from threading import Thread, Lock - -import time - -from htpclient.config import Config -from htpclient.hashcat_status import HashcatStatus -from htpclient.initialize import Initialize -from htpclient.jsonRequest import JsonRequest, os -from htpclient.helpers import send_error, update_files, kill_hashcat, get_bit, print_speed, get_rules_and_hl, get_wordlist, escape_ansi -from htpclient.dicts import * - - -class HashcatCracker: - def __init__(self, cracker_id, binary_download): - self.config = Config() - self.io_q = Queue() - - # Build cracker executable name by taking basename plus extension - self.executable_name = binary_download.get_version()['executable'] - k = self.executable_name.rfind(".") - self.executable_name = self.executable_name[:k] + "." + self.executable_name[k + 1:] - self.cracker_path = "crackers/" + str(cracker_id) + "/" - self.callPath = self.executable_name - if Initialize.get_os() != 1: - self.callPath = "./" + self.callPath - - if not os.path.isfile(self.cracker_path + self.callPath): # in case it's not the new hashcat filename, try the old one (hashcat.) - self.executable_name = binary_download.get_version()['executable'] - k = self.executable_name.rfind(".") - self.executable_name = self.executable_name[:k] + get_bit() + "." + self.executable_name[k + 1:] - self.cracker_path = "crackers/" + str(cracker_id) + "/" - self.callPath = self.executable_name - if Initialize.get_os() != 1: - self.callPath = "./" + self.callPath - - self.lock = Lock() - self.cracks = [] - self.first_status = False - self.usePipe = False - self.progressVal = 0 - self.statusCount = 0 - self.last_update = 0 - self.uses_slow_hash_flag = False - self.wasStopped = False - - def build_command(self, task, chunk): - args = " --machine-readable --quiet --status --restore-disable --session=hashtopolis" - args += " --status-timer " + str(task['statustimer']) - args += " --outfile-check-timer=" + str(task['statustimer']) - args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) - args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=15 -p \"" + str(chr(9)) + "\"" - args += " -s " + str(chunk['skip']) - args += " -l " + str(chunk['length']) - if 'useBrain' in task and task['useBrain']: # when using brain we set the according parameters - args += " --brain-client --brain-host " + task['brainHost'] - args += " --brain-port " + str(task['brainPort']) - args += " --brain-password " + task['brainPass'] - if 'brainFeatures' in task: - args += " --brain-client-features " + str(task['brainFeatures']) - else: # remove should only be used if we run without brain - args += " --potfile-disable --remove --remove-timer=" + str(task['statustimer']) - args += " " + update_files(task['attackcmd']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + " " + task['cmdpars'] - if args.find(" -S") != -1: - self.uses_slow_hash_flag = True - return self.callPath + args - - def build_pipe_command(self, task, chunk): - # call the command with piping - pre_args = " --stdout -s " + str(chunk['skip']) + " -l " + str(chunk['length']) + ' ' - pre_args += update_files(task['attackcmd']).replace(task['hashlistAlias'], '') - post_args = " --machine-readable --quiet --status --remove --restore-disable --potfile-disable --session=hashtopolis" - post_args += " --status-timer " + str(task['statustimer']) - post_args += " --outfile-check-timer=" + str(task['statustimer']) - post_args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) - post_args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=15 -p \"" + str(chr(9)) + "\"" - post_args += " --remove-timer=" + str(task['statustimer']) - post_args += " ../../hashlists/" + str(task['hashlistId']) - return self.callPath + pre_args + " | " + self.callPath + post_args + task['cmdpars'] - - def build_prince_command(self, task, chunk): - binary = "../../prince/pp64." - if Initialize.get_os() != 1: - binary = "./" + binary + "bin" - else: - binary += "exe" - pre_args = " -s " + str(chunk['skip']) + " -l " + str(chunk['length']) + ' ' - pre_args += get_wordlist(update_files(task['attackcmd']).replace(task['hashlistAlias'], '')) - post_args = " --machine-readable --quiet --status --remove --restore-disable --potfile-disable --session=hashtopolis" - post_args += " --status-timer " + str(task['statustimer']) - post_args += " --outfile-check-timer=" + str(task['statustimer']) - post_args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) - post_args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=15 -p \"" + str(chr(9)) + "\"" - post_args += " --remove-timer=" + str(task['statustimer']) - post_args += " ../../hashlists/" + str(task['hashlistId']) - post_args += get_rules_and_hl(update_files(task['attackcmd']), task['hashlistAlias']).replace(task['hashlistAlias'], '') - return binary + pre_args + " | " + self.callPath + post_args + task['cmdpars'] - - def run_chunk(self, task, chunk): - if 'enforcePipe' in task and task['enforcePipe']: - logging.info("Enforcing pipe command because of task setting...") - self.usePipe = True - if task['usePrince']: - full_cmd = self.build_prince_command(task, chunk) - elif self.usePipe: - full_cmd = self.build_pipe_command(task, chunk) - else: - full_cmd = self.build_command(task, chunk) - self.statusCount = 0 - self.wasStopped = False - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - # clear old found file - earlier we deleted them, but just in case, we just move it to a unique filename if configured so - if os.path.exists("hashlists/" + str(task['hashlistId']) + ".out"): - if self.config.get_value('outfile-history'): - os.rename("hashlists/" + str(task['hashlistId']) + ".out", "hashlists/" + str(task['hashlistId']) + "_" + str(time.time()) + ".out") - else: - os.unlink("hashlists/" + str(task['hashlistId']) + ".out") - # create zap folder - if not os.path.exists("hashlist_" + str(task['hashlistId'])): - os.mkdir("hashlist_" + str(task['hashlistId'])) - logging.debug("CALL: " + full_cmd) - if Initialize.get_os() != 1: - process = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path, preexec_fn=os.setsid) - else: - process = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) - - logging.debug("started cracking") - out_thread = Thread(target=self.stream_watcher, name='stdout-watcher', args=('OUT', process.stdout)) - err_thread = Thread(target=self.stream_watcher, name='stderr-watcher', args=('ERR', process.stderr)) - crk_thread = Thread(target=self.output_watcher, name='crack-watcher', args=("hashlists/" + str(task['hashlistId']) + ".out", process)) - out_thread.start() - err_thread.start() - crk_thread.start() - self.first_status = False - self.last_update = time.time() - - main_thread = Thread(target=self.run_loop, name='run_loop', args=(process, chunk, task)) - main_thread.start() - - # wait for all threads to finish - process.wait() - crk_thread.join() - out_thread.join() - err_thread.join() - main_thread.join() - logging.info("finished chunk") - - def run_loop(self, proc, chunk, task): - self.cracks = [] - piping_threshold = 95 - enable_piping = True - if self.config.get_value('piping-threshold'): - piping_threshold = self.config.get_value('piping-threshold') - if self.config.get_value('allow-piping') != '': - enable_piping = self.config.get_value('allow-piping') - while True: - try: - # Block for 1 second. - if not self.first_status and self.last_update < time.time() - 5: - # send update - query = copy_and_set_token(dict_sendProgress, self.config.get_value('token')) - query['chunkId'] = chunk['chunkId'] - query['keyspaceProgress'] = chunk['skip'] - query['relativeProgress'] = 0 - query['speed'] = 0 - query['state'] = 2 - query['cracks'] = [] - req = JsonRequest(query) - logging.info("Sending keepalive progress to avoid timeout...") - req.execute() - self.last_update = time.time() - item = self.io_q.get(True, 1) - except Empty: - # No output in either streams for a second. Are we done? - if proc.poll() is not None: - # is the case when the process is finished - break - else: - identifier, line = item - if identifier == 'OUT': - status = HashcatStatus(line.decode()) - if status.is_valid(): - self.statusCount += 1 - - # test if we have a low utility - # not allowed if brain is used - if enable_piping and not self.uses_slow_hash_flag and ('useBrain' not in task or not task['useBrain']) and 'slowHash' in task and task['slowHash'] and not self.usePipe: - if task['files'] and not task['usePrince'] and 1 < self.statusCount < 10 and status.get_util() != -1 and status.get_util() < piping_threshold: - # we need to try piping -> kill the process and then wait for issuing the chunk again - self.usePipe = True - chunk_start = int(status.get_progress_total() / (chunk['skip'] + chunk['length']) * chunk['skip']) - self.progressVal = status.get_progress_total() - chunk_start - logging.info("Detected low UTIL value, restart chunk with piping...") - try: - kill_hashcat(proc.pid, Initialize.get_os()) - except ProcessLookupError: - pass - return - - self.first_status = True - # send update to server - logging.debug(line.decode().replace('\n', '').replace('\r', '')) - total = status.get_progress_total() - if self.usePipe: # if we are piping, we might have saved the total progress before switching to piping, so we can use this - total = self.progressVal - # we need to calculate the chunk start, because progress does not start at 0 for a chunk - chunk_start = int(status.get_progress_total() / (chunk['skip'] + chunk['length']) * chunk['skip']) - if total > 0: - relative_progress = int((status.get_progress() - chunk_start) / float(total - chunk_start) * 10000) - else: # this is the case when we cannot say anything about the progress - relative_progress = 0 - speed = status.get_speed() - initial = True - if status.get_state() == 4 or status.get_state() == 5: - time.sleep(5) # we wait five seconds so all output is loaded from file - # reset piping stuff when a chunk is successfully finished - self.progressVal = 0 - self.usePipe = False - while self.cracks or initial: - self.lock.acquire() - initial = False - cracks_backup = [] - if len(self.cracks) > 1000: - # we split - cnt = 0 - new_cracks = [] - for crack in self.cracks: - cnt += 1 - if cnt > 1000: - cracks_backup.append(crack) - else: - new_cracks.append(crack) - self.cracks = new_cracks - query = copy_and_set_token(dict_sendProgress, self.config.get_value('token')) - query['chunkId'] = chunk['chunkId'] - query['keyspaceProgress'] = status.get_curku() - if (self.usePipe or task['usePrince']) and status.get_curku() == 0: - query['keyspaceProgress'] = chunk['skip'] - query['relativeProgress'] = relative_progress - query['speed'] = speed - query['state'] = status.get_state() - # crack format: hash[:salt]:plain:hex_plain:crack_pos (separator will be tab instead of :) - prepared = [] - for crack in self.cracks: - prepared.append(crack.split("\t")) - query['cracks'] = prepared - if status.get_temps(): - query['gpuTemp'] = status.get_temps() - if status.get_all_util(): - query['gpuUtil'] = status.get_all_util() - req = JsonRequest(query) - - logging.debug("Sending " + str(len(self.cracks)) + " cracks...") - ans = req.execute() - if ans is None: - logging.error("Failed to send solve!") - elif ans['response'] != 'SUCCESS': - self.wasStopped = True - logging.error("Error from server on solve: " + str(ans)) - try: - kill_hashcat(proc.pid, Initialize.get_os()) - except ProcessLookupError: - pass - sleep(5) - return - elif 'agent' in ans.keys() and ans['agent'] == 'stop': - # server set agent to stop - self.wasStopped = True - logging.info("Received stop order from server!") - try: - kill_hashcat(proc.pid, Initialize.get_os()) - except ProcessLookupError: - pass - sleep(5) - return - else: - cracks_count = len(self.cracks) - self.cracks = cracks_backup - zaps = ans['zaps'] - if zaps: - logging.debug("Writing zaps") - zap_output = "\tFF\n".join(zaps) + '\tFF\n' - f = open("hashlist_" + str(task['hashlistId']) + "/" + str(time.time()), 'a') - f.write(zap_output) - f.close() - logging.info("Progress:" + str("{:6.2f}".format(relative_progress / 100)) + "% Speed: " + print_speed(speed) + " Cracks: " + str(cracks_count) + " Accepted: " + str(ans['cracked']) + " Skips: " + str(ans['skipped']) + " Zaps: " + str(len(zaps))) - self.lock.release() - else: - # hacky solution to exclude warnings from hashcat - if str(line[0]) not in string.printable: - continue - else: - pass # logging.warning("HCOUT: " + line.strip()) - elif identifier == 'ERR': - msg = escape_ansi(line.replace(b"\r\n", b"\n").decode('utf-8')).strip() - if msg and str(msg) != '^C': # this is maybe not the fanciest way, but as ctrl+c is sent to the underlying process it reports it to stderr - logging.error("HC error: " + msg) - send_error(msg, self.config.get_value('token'), task['taskId'], chunk['chunkId']) - sleep(0.1) # we set a minimal sleep to avoid overreaction of the client sending a huge number of errors, but it should not be slowed down too much, in case the errors are not critical and the agent can continue - - def measure_keyspace(self, task, chunk): - if task['usePrince']: - return self.prince_keyspace(task, chunk) - full_cmd = self.callPath + " --keyspace --quiet " + update_files(task['attackcmd']).replace(task['hashlistAlias'] + " ", "") + ' ' + task['cmdpars'] - if 'useBrain' in task and task['useBrain']: - full_cmd += " -S" - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - try: - logging.debug("CALL: " + full_cmd) - output = subprocess.check_output(full_cmd, shell=True, cwd=self.cracker_path) - except subprocess.CalledProcessError as e: - logging.error("Error during keyspace measure: " + str(e)) - send_error("Keyspace measure failed!", self.config.get_value('token'), task['taskId'], None) - sleep(5) - return False - output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") - keyspace = "0" - for line in output: - if not line: - continue - keyspace = line - return chunk.send_keyspace(int(keyspace), task['taskId']) - - def prince_keyspace(self, task, chunk): - binary = "pp64." - if Initialize.get_os() != 1: - binary = "./" + binary + "bin" - else: - binary += "exe" - full_cmd = binary + " --keyspace " + get_wordlist(update_files(task['attackcmd'], True).replace(task['hashlistAlias'], "")) - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - try: - logging.debug("CALL: " + full_cmd) - output = subprocess.check_output(full_cmd, shell=True, cwd="prince") - except subprocess.CalledProcessError: - logging.error("Error during PRINCE keyspace measure") - send_error("PRINCE keyspace measure failed!", self.config.get_value('token'), task['taskId'], None) - sleep(5) - return False - output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") - keyspace = "0" - for line in output: - if not line: - continue - keyspace = line - # as the keyspace of prince can get very very large, we only save it in case it's small enough to fit in a long, - # otherwise we assume that the user will abort the task earlier anyway - if int(keyspace) > 9000000000000000000: # close to max size of a long long int - return chunk.send_keyspace(-1, task['taskId']) - return chunk.send_keyspace(int(keyspace), task['taskId']) - - def run_benchmark(self, task): - if task['benchType'] == 'speed': - # do a speed benchmark - return self.run_speed_benchmark(task) - - args = " --machine-readable --quiet --runtime=" + str(task['bench']) - args += " --restore-disable --potfile-disable --session=hashtopolis -p \"" + str(chr(9)) + "\" " - args += update_files(task['attackcmd']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + ' ' + task['cmdpars'] - args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out" - full_cmd = self.callPath + args - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - logging.debug("CALL: " + full_cmd) - proc = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) - output, error = proc.communicate() - logging.debug("started benchmark") - proc.wait() # wait until done - if error: - error = escape_ansi(error.replace(b"\r\n", b"\n").decode('utf-8')) - # parse errors and send it to server - error = error.split('\n') - for line in error: - if not line: - continue - query = copy_and_set_token(dict_clientError, self.config.get_value('token')) - query['taskId'] = task['taskId'] - query['message'] = line - req = JsonRequest(query) - req.execute() - # return 0 it might not be ideal to return here. In case of errors still try to read the benchmark. - if output: - output = output.replace(b"\r\n", b"\n").decode('utf-8') - output = output.split('\n') - last_valid_status = None - for line in output: - if not line: - continue - logging.debug("HCSTAT: " + line.strip()) - status = HashcatStatus(line) - if status.is_valid(): - last_valid_status = status - if last_valid_status is None: - return 0 - # we just calculate how far in the task the agent went during the benchmark time - return (last_valid_status.get_progress() - last_valid_status.get_rejected()) / float(last_valid_status.get_progress_total()) - return 0 - - def stream_watcher(self, identifier, stream): - for line in stream: - self.io_q.put((identifier, line)) - if not stream.closed: - stream.close() - - def run_speed_benchmark(self, task): - args = " --machine-readable --quiet --progress-only" - args += " --restore-disable --potfile-disable --session=hashtopolis -p \"" + str(chr(9)) + "\" " - if task['usePrince']: - args += get_rules_and_hl(update_files(task['attackcmd']), task['hashlistAlias']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + ' ' - args += " example.dict" + ' ' + task['cmdpars'] - else: - args += update_files(task['attackcmd']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + ' ' + task['cmdpars'] - if 'useBrain' in task and task['useBrain']: - args += " -S" - args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out" - full_cmd = self.callPath + args - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - try: - logging.debug("CALL: " + full_cmd) - output = subprocess.check_output(full_cmd, shell=True, cwd=self.cracker_path) - except subprocess.CalledProcessError as e: - logging.error("Error during speed benchmark, return code: " + str(e.returncode)) - send_error("Speed benchmark failed!", self.config.get_value('token'), task['taskId'], None) - return 0 - output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") - benchmark_sum = [0, 0] - for line in output: - if not line: - continue - line = line.split(":") - if len(line) != 3: - continue - # we need to do a weighted sum of all the time outputs of the GPUs - benchmark_sum[0] += int(line[1]) - benchmark_sum[1] += float(line[2])*int(line[1]) - return str(benchmark_sum[0]) + ":" + str(float(benchmark_sum[1]) / benchmark_sum[0]) - - def output_watcher(self, file_path, process): - while not os.path.exists(file_path): - if process.poll() is not None: - return - time.sleep(1) - file_handle = open(file_path, encoding="utf-8") - end_count = 0 - while 1: - where = file_handle.tell() - line = file_handle.readline() - if not line: - if process.poll() is None: - time.sleep(0.05) - file_handle.seek(where) - else: - time.sleep(0.05) - end_count += 1 - if end_count > 20: - break - else: - self.lock.acquire() - self.cracks.append(line.strip()) - self.lock.release() - file_handle.close() - - def agent_stopped(self): - return self.wasStopped - - def run_health_check(self, attack, hashlist_alias): - args = " --machine-readable --quiet" - args += " --restore-disable --potfile-disable --session=health " - args += update_files(attack).replace(hashlist_alias, "../../hashlists/health_check.txt") - args += " -o ../../hashlists/health_check.out" - full_cmd = self.callPath + args - if Initialize.get_os() == 1: - full_cmd = full_cmd.replace("/", '\\') - logging.debug("CALL: " + full_cmd) - proc = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) - output, error = proc.communicate() - logging.debug("Started health check attack") - # wait until done, on the health check we don't send any update during running. Maybe later we could at least - # introduce some heartbeat update to make visible that the agent is still alive. - proc.wait() - errors = [] - states = [] - if error: - error = escape_ansi(error.replace(b"\r\n", b"\n").decode('utf-8')) - error = error.split('\n') - for line in error: - if not line: - continue - errors.append(line) - if output: - output = escape_ansi(output.replace(b"\r\n", b"\n").decode('utf-8')) - output = output.split('\n') - for line in output: - if not line: - continue - logging.debug(line) - status = HashcatStatus(line) - if status.is_valid(): - states.append(status) - return [states, errors] +import string +import logging +import subprocess +from time import sleep +from queue import Queue, Empty +from threading import Thread, Lock + +import time + +from htpclient.config import Config +from htpclient.hashcat_status import HashcatStatus +from htpclient.initialize import Initialize +from htpclient.jsonRequest import JsonRequest, os +from htpclient.helpers import send_error, update_files, kill_hashcat, get_bit, print_speed, get_rules_and_hl, get_wordlist, escape_ansi +from htpclient.dicts import * + + +class HashcatCracker: + def __init__(self, cracker_id, binary_download): + self.config = Config() + self.io_q = Queue() + self.version_string = "" + + # Build cracker executable name by taking basename plus extension + self.executable_name = binary_download.get_version()['executable'] + k = self.executable_name.rfind(".") + self.executable_name = self.executable_name[:k] + "." + self.executable_name[k + 1:] + self.cracker_path = "crackers/" + str(cracker_id) + "/" + self.callPath = self.executable_name + if Initialize.get_os() != 1: + self.callPath = "./" + self.callPath + + if not os.path.isfile(self.cracker_path + self.callPath): # in case it's not the new hashcat filename, try the old one (hashcat.) + self.executable_name = binary_download.get_version()['executable'] + k = self.executable_name.rfind(".") + self.executable_name = self.executable_name[:k] + get_bit() + "." + self.executable_name[k + 1:] + self.cracker_path = "crackers/" + str(cracker_id) + "/" + self.callPath = self.executable_name + if Initialize.get_os() != 1: + self.callPath = "./" + self.callPath + + cmd = self.callPath + " --version" + output = '' + try: + logging.debug("CALL: " + cmd) + output = subprocess.check_output(cmd, shell=True, cwd=self.cracker_path) + except subprocess.CalledProcessError as e: + logging.error("Error during version detection: " + str(e)) + sleep(5) + self.version_string = output.decode().replace('v', '') + + self.lock = Lock() + self.cracks = [] + self.first_status = False + self.usePipe = False + self.progressVal = 0 + self.statusCount = 0 + self.last_update = 0 + self.uses_slow_hash_flag = False + self.wasStopped = False + + def get_outfile_format(self): + if self.version_string.find('-') == -1: + return "15" # if we cannot determine the version, we will use the old format + split = self.version_string.split('-') + if len(split) < 2: + return "15" # something is wrong with the version string, go for old format + release = str(split[0]).split('.') + commit = str(split[1]) + if int(str(release[0])) < 5: + return "15" + elif int(str(release[0])) == 5 and int(str(release[1])) < 1: + return "15" + elif int(str(release[0])) == 5 and int(str(release[1])) == 1 and int(str(release[2])) == 0 and int(commit) < 1618: + return "15" + return "1,2,3,4" # new outfile format + + def build_command(self, task, chunk): + args = " --machine-readable --quiet --status --restore-disable --session=hashtopolis" + args += " --status-timer " + str(task['statustimer']) + args += " --outfile-check-timer=" + str(task['statustimer']) + args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) + args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=" + self.get_outfile_format() + " -p \"" + str(chr(9)) + "\"" + args += " -s " + str(chunk['skip']) + args += " -l " + str(chunk['length']) + if 'useBrain' in task and task['useBrain']: # when using brain we set the according parameters + args += " --brain-client --brain-host " + task['brainHost'] + args += " --brain-port " + str(task['brainPort']) + args += " --brain-password " + task['brainPass'] + if 'brainFeatures' in task: + args += " --brain-client-features " + str(task['brainFeatures']) + else: # remove should only be used if we run without brain + args += " --potfile-disable --remove --remove-timer=" + str(task['statustimer']) + args += " " + update_files(task['attackcmd']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + " " + task['cmdpars'] + if args.find(" -S") != -1: + self.uses_slow_hash_flag = True + return self.callPath + args + + def build_pipe_command(self, task, chunk): + # call the command with piping + pre_args = " --stdout -s " + str(chunk['skip']) + " -l " + str(chunk['length']) + ' ' + pre_args += update_files(task['attackcmd']).replace(task['hashlistAlias'], '') + post_args = " --machine-readable --quiet --status --remove --restore-disable --potfile-disable --session=hashtopolis" + post_args += " --status-timer " + str(task['statustimer']) + post_args += " --outfile-check-timer=" + str(task['statustimer']) + post_args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) + post_args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=" + self.get_outfile_format() + " -p \"" + str(chr(9)) + "\"" + post_args += " --remove-timer=" + str(task['statustimer']) + post_args += " ../../hashlists/" + str(task['hashlistId']) + return self.callPath + pre_args + " | " + self.callPath + post_args + task['cmdpars'] + + # DEPRECATED + def build_prince_command(self, task, chunk): + binary = "../../prince/pp64." + if Initialize.get_os() != 1: + binary = "./" + binary + "bin" + else: + binary += "exe" + pre_args = " -s " + str(chunk['skip']) + " -l " + str(chunk['length']) + ' ' + pre_args += get_wordlist(update_files(task['attackcmd']).replace(task['hashlistAlias'], '')) + post_args = " --machine-readable --quiet --status --remove --restore-disable --potfile-disable --session=hashtopolis" + post_args += " --status-timer " + str(task['statustimer']) + post_args += " --outfile-check-timer=" + str(task['statustimer']) + post_args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) + post_args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=" + self.get_outfile_format() + " -p \"" + str(chr(9)) + "\"" + post_args += " --remove-timer=" + str(task['statustimer']) + post_args += " ../../hashlists/" + str(task['hashlistId']) + post_args += get_rules_and_hl(update_files(task['attackcmd']), task['hashlistAlias']).replace(task['hashlistAlias'], '') + return binary + pre_args + " | " + self.callPath + post_args + task['cmdpars'] + + def build_preprocessor_command(self, task, chunk, preprocessor): + binary = "../../preprocessor/" + str(task['preprocessor']) + "/" + preprocessor['executable'] + if Initialize.get_os() != 1: + binary = "./" + binary + if not os.path.isfile(binary): + split = binary.split(".") + binary = '.'.join(split[:-1]) + get_bit() + "." + split[-1] + + # in case the skip or limit command are not available, we try to achieve the same with head/tail (the more chunks are run, the more inefficient it might be) + if preprocessor['skipCommand'] is not None and preprocessor['limitCommand'] is not None: + pre_args = " " + preprocessor['skipCommand'] + " " + str(chunk['skip']) + " " + preprocessor['limitCommand'] + " " + str(chunk['length']) + ' ' + else: + pre_args = "" + + pre_args += ' ' + update_files(task['preprocessorCommand']) + + # TODO: add support for windows as well (pre-built tools) + if preprocessor['skipCommand'] is None or preprocessor['limitCommand'] is None: + pre_args += " | head -n " + str(chunk['skip'] + chunk['length']) + " | tail -n " + str(chunk['length']) + + post_args = " --machine-readable --quiet --status --remove --restore-disable --potfile-disable --session=hashtopolis" + post_args += " --status-timer " + str(task['statustimer']) + post_args += " --outfile-check-timer=" + str(task['statustimer']) + post_args += " --outfile-check-dir=../../hashlist_" + str(task['hashlistId']) + post_args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out --outfile-format=" + self.get_outfile_format() + " -p \"" + str(chr(9)) + "\"" + post_args += " --remove-timer=" + str(task['statustimer']) + post_args += " ../../hashlists/" + str(task['hashlistId']) + post_args += update_files(task['attackcmd']).replace(task['hashlistAlias'], '') + return binary + pre_args + " | " + self.callPath + post_args + task['cmdpars'] + + def run_chunk(self, task, chunk, preprocessor): + if 'enforcePipe' in task and task['enforcePipe']: + logging.info("Enforcing pipe command because of task setting...") + self.usePipe = True + if 'usePrince' in task and task['usePrince']: # DEPRECATED + full_cmd = self.build_prince_command(task, chunk) + elif 'usePreprocessor' in task and task['usePreprocessor']: + full_cmd = self.build_preprocessor_command(task, chunk, preprocessor) + elif self.usePipe: + full_cmd = self.build_pipe_command(task, chunk) + else: + full_cmd = self.build_command(task, chunk) + self.statusCount = 0 + self.wasStopped = False + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + # clear old found file - earlier we deleted them, but just in case, we just move it to a unique filename if configured so + if os.path.exists("hashlists/" + str(task['hashlistId']) + ".out"): + if self.config.get_value('outfile-history'): + os.rename("hashlists/" + str(task['hashlistId']) + ".out", "hashlists/" + str(task['hashlistId']) + "_" + str(time.time()) + ".out") + else: + os.unlink("hashlists/" + str(task['hashlistId']) + ".out") + # create zap folder + if not os.path.exists("hashlist_" + str(task['hashlistId'])): + os.mkdir("hashlist_" + str(task['hashlistId'])) + logging.debug("CALL: " + full_cmd) + if Initialize.get_os() != 1: + process = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path, preexec_fn=os.setsid) + else: + process = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) + + logging.debug("started cracking") + out_thread = Thread(target=self.stream_watcher, name='stdout-watcher', args=('OUT', process.stdout)) + err_thread = Thread(target=self.stream_watcher, name='stderr-watcher', args=('ERR', process.stderr)) + crk_thread = Thread(target=self.output_watcher, name='crack-watcher', args=("hashlists/" + str(task['hashlistId']) + ".out", process)) + out_thread.start() + err_thread.start() + crk_thread.start() + self.first_status = False + self.last_update = time.time() + + main_thread = Thread(target=self.run_loop, name='run_loop', args=(process, chunk, task)) + main_thread.start() + + # wait for all threads to finish + process.wait() + crk_thread.join() + out_thread.join() + err_thread.join() + main_thread.join() + logging.info("finished chunk") + + def run_loop(self, proc, chunk, task): + self.cracks = [] + piping_threshold = 95 + enable_piping = True + if self.config.get_value('piping-threshold'): + piping_threshold = self.config.get_value('piping-threshold') + if self.config.get_value('allow-piping') != '': + enable_piping = self.config.get_value('allow-piping') + while True: + try: + # Block for 1 second. + if not self.first_status and self.last_update < time.time() - 5: + # send update + query = copy_and_set_token(dict_sendProgress, self.config.get_value('token')) + query['chunkId'] = chunk['chunkId'] + query['keyspaceProgress'] = chunk['skip'] + query['relativeProgress'] = 0 + query['speed'] = 0 + query['state'] = 2 + query['cracks'] = [] + req = JsonRequest(query) + logging.info("Sending keepalive progress to avoid timeout...") + req.execute() + self.last_update = time.time() + item = self.io_q.get(True, 1) + except Empty: + # No output in either streams for a second. Are we done? + if proc.poll() is not None: + # is the case when the process is finished + break + else: + identifier, line = item + if identifier == 'OUT': + status = HashcatStatus(line.decode()) + if status.is_valid(): + self.statusCount += 1 + + # test if we have a low utility + # not allowed if brain is used + if enable_piping and not self.uses_slow_hash_flag and ('useBrain' not in task or not task['useBrain']) and 'slowHash' in task and task['slowHash'] and not self.usePipe: + if task['files'] and not ('usePrince' in task and task['usePrince']) and not ('usePreprocessor' in task and task['usePreprocessor']) and 1 < self.statusCount < 10 and status.get_util() != -1 and status.get_util() < piping_threshold: + # we need to try piping -> kill the process and then wait for issuing the chunk again + self.usePipe = True + chunk_start = int(status.get_progress_total() / (chunk['skip'] + chunk['length']) * chunk['skip']) + self.progressVal = status.get_progress_total() - chunk_start + logging.info("Detected low UTIL value, restart chunk with piping...") + try: + kill_hashcat(proc.pid, Initialize.get_os()) + except ProcessLookupError: + pass + return + + self.first_status = True + # send update to server + logging.debug(line.decode().replace('\n', '').replace('\r', '')) + total = status.get_progress_total() + if self.usePipe: # if we are piping, we might have saved the total progress before switching to piping, so we can use this + total = self.progressVal + # we need to calculate the chunk start, because progress does not start at 0 for a chunk + chunk_start = int(status.get_progress_total() / (chunk['skip'] + chunk['length']) * chunk['skip']) + if total > 0: + relative_progress = int((status.get_progress() - chunk_start) / float(total - chunk_start) * 10000) + else: # this is the case when we cannot say anything about the progress + relative_progress = 0 + speed = status.get_speed() + initial = True + if status.get_state() == 4 or status.get_state() == 5: + time.sleep(5) # we wait five seconds so all output is loaded from file + # reset piping stuff when a chunk is successfully finished + self.progressVal = 0 + self.usePipe = False + while self.cracks or initial: + self.lock.acquire() + initial = False + cracks_backup = [] + if len(self.cracks) > 1000: + # we split + cnt = 0 + new_cracks = [] + for crack in self.cracks: + cnt += 1 + if cnt > 1000: + cracks_backup.append(crack) + else: + new_cracks.append(crack) + self.cracks = new_cracks + query = copy_and_set_token(dict_sendProgress, self.config.get_value('token')) + query['chunkId'] = chunk['chunkId'] + query['keyspaceProgress'] = status.get_curku() + if (self.usePipe or 'usePrince' in task and task['usePrince'] or 'usePreprocessor' in task and task['usePreprocessor']) and status.get_curku() == 0: + query['keyspaceProgress'] = chunk['skip'] + query['relativeProgress'] = relative_progress + query['speed'] = speed + query['state'] = status.get_state() + # crack format: hash[:salt]:plain:hex_plain:crack_pos (separator will be tab instead of :) + prepared = [] + for crack in self.cracks: + prepared.append(crack.split("\t")) + query['cracks'] = prepared + if status.get_temps(): + query['gpuTemp'] = status.get_temps() + if status.get_all_util(): + query['gpuUtil'] = status.get_all_util() + req = JsonRequest(query) + + logging.debug("Sending " + str(len(self.cracks)) + " cracks...") + ans = req.execute() + if ans is None: + logging.error("Failed to send solve!") + elif ans['response'] != 'SUCCESS': + self.wasStopped = True + logging.error("Error from server on solve: " + str(ans)) + try: + kill_hashcat(proc.pid, Initialize.get_os()) + except ProcessLookupError: + pass + sleep(5) + return + elif 'agent' in ans.keys() and ans['agent'] == 'stop': + # server set agent to stop + self.wasStopped = True + logging.info("Received stop order from server!") + try: + kill_hashcat(proc.pid, Initialize.get_os()) + except ProcessLookupError: + pass + sleep(5) + return + else: + cracks_count = len(self.cracks) + self.cracks = cracks_backup + zaps = ans['zaps'] + if zaps: + logging.debug("Writing zaps") + zap_output = "\tFF\n".join(zaps) + '\tFF\n' + f = open("hashlist_" + str(task['hashlistId']) + "/" + str(time.time()), 'a') + f.write(zap_output) + f.close() + logging.info("Progress:" + str("{:6.2f}".format(relative_progress / 100)) + "% Speed: " + print_speed(speed) + " Cracks: " + str(cracks_count) + " Accepted: " + str(ans['cracked']) + " Skips: " + str(ans['skipped']) + " Zaps: " + str(len(zaps))) + self.lock.release() + else: + # hacky solution to exclude warnings from hashcat + if str(line[0]) not in string.printable: + continue + else: + pass # logging.warning("HCOUT: " + line.strip()) + elif identifier == 'ERR': + msg = escape_ansi(line.replace(b"\r\n", b"\n").decode('utf-8')).strip() + if msg and str(msg) != '^C': # this is maybe not the fanciest way, but as ctrl+c is sent to the underlying process it reports it to stderr + logging.error("HC error: " + msg) + send_error(msg, self.config.get_value('token'), task['taskId'], chunk['chunkId']) + sleep(0.1) # we set a minimal sleep to avoid overreaction of the client sending a huge number of errors, but it should not be slowed down too much, in case the errors are not critical and the agent can continue + + def measure_keyspace(self, task, chunk): + if 'usePrince' in task.get_task() and task.get_task()['usePrince']: + return self.prince_keyspace(task.get_task(), chunk) + elif 'usePreprocessor' in task.get_task() and task.get_task()['usePreprocessor']: + return self.preprocessor_keyspace(task, chunk) + task = task.get_task() # TODO: refactor this to be better code + full_cmd = self.callPath + " --keyspace --quiet " + update_files(task['attackcmd']).replace(task['hashlistAlias'] + " ", "") + ' ' + task['cmdpars'] + if 'useBrain' in task and task['useBrain']: + full_cmd += " -S" + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + try: + logging.debug("CALL: " + full_cmd) + output = subprocess.check_output(full_cmd, shell=True, cwd=self.cracker_path) + except subprocess.CalledProcessError as e: + logging.error("Error during keyspace measure: " + str(e)) + send_error("Keyspace measure failed!", self.config.get_value('token'), task['taskId'], None) + sleep(5) + return False + output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") + keyspace = "0" + for line in output: + if not line: + continue + keyspace = line + return chunk.send_keyspace(int(keyspace), task['taskId']) + + # DEPRECATED + def prince_keyspace(self, task, chunk): + binary = "pp64." + if Initialize.get_os() != 1: + binary = "./" + binary + "bin" + else: + binary += "exe" + full_cmd = binary + " --keyspace " + get_wordlist(update_files(task['attackcmd'], True).replace(task['hashlistAlias'], "")) + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + try: + logging.debug("CALL: " + full_cmd) + output = subprocess.check_output(full_cmd, shell=True, cwd="prince") + except subprocess.CalledProcessError: + logging.error("Error during PRINCE keyspace measure") + send_error("PRINCE keyspace measure failed!", self.config.get_value('token'), task['taskId'], None) + sleep(5) + return False + output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") + keyspace = "0" + for line in output: + if not line: + continue + keyspace = line + # as the keyspace of prince can get very very large, we only save it in case it's small enough to fit in a long, + # otherwise we assume that the user will abort the task earlier anyway + if int(keyspace) > 9000000000000000000: # close to max size of a long long int + return chunk.send_keyspace(-1, task['taskId']) + return chunk.send_keyspace(int(keyspace), task['taskId']) + + def preprocessor_keyspace(self, task, chunk): + preprocessor = task.get_preprocessor() + if preprocessor['keyspaceCommand'] is None: # in case there is no keyspace flag, we just assume the task will be that large to run forever + return chunk.send_keyspace(-1, task.get_task()['taskId']) + + binary = preprocessor['executable'] + if Initialize.get_os() != 1: + binary = "./" + binary + if not os.path.isfile(binary): + split = binary.split(".") + binary = '.'.join(split[:-1]) + get_bit() + "." + split[-1] + + full_cmd = binary + " " + preprocessor['keyspaceCommand'] + " " + update_files(task.get_task()['preprocessorCommand']) + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + try: + logging.debug("CALL: " + full_cmd) + output = subprocess.check_output(full_cmd, shell=True, cwd="preprocessor/" + str(task.get_task()['preprocessor'])) + except subprocess.CalledProcessError: + logging.error("Error during preprocessor keyspace measure") + send_error("Preprocessor keyspace measure failed!", self.config.get_value('token'), task.get_task()['taskId'], None) + sleep(5) + return False + output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") + keyspace = "0" + for line in output: + if not line: + continue + keyspace = line + # as the keyspace of preprocessors can get very very large, we only save it in case it's small enough to fit in a long, + # otherwise we assume that the user will abort the task earlier anyway + if int(keyspace) > 9000000000000000000: # close to max size of a long long int + return chunk.send_keyspace(-1, task.get_task()['taskId']) + return chunk.send_keyspace(int(keyspace), task.get_task()['taskId']) + + def run_benchmark(self, task): + if task['benchType'] == 'speed': + # do a speed benchmark + return self.run_speed_benchmark(task) + + args = " --machine-readable --quiet --runtime=" + str(task['bench']) + args += " --restore-disable --potfile-disable --session=hashtopolis -p \"" + str(chr(9)) + "\" " + args += update_files(task['attackcmd']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + ' ' + task['cmdpars'] + args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out" + full_cmd = self.callPath + args + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + logging.debug("CALL: " + full_cmd) + proc = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) + output, error = proc.communicate() + logging.debug("started benchmark") + proc.wait() # wait until done + if error: + error = escape_ansi(error.replace(b"\r\n", b"\n").decode('utf-8')) + # parse errors and send it to server + error = error.split('\n') + for line in error: + if not line: + continue + query = copy_and_set_token(dict_clientError, self.config.get_value('token')) + query['taskId'] = task['taskId'] + query['message'] = line + req = JsonRequest(query) + req.execute() + # return 0 it might not be ideal to return here. In case of errors still try to read the benchmark. + if output: + output = output.replace(b"\r\n", b"\n").decode('utf-8') + output = output.split('\n') + last_valid_status = None + for line in output: + if not line: + continue + logging.debug("HCSTAT: " + line.strip()) + status = HashcatStatus(line) + if status.is_valid(): + last_valid_status = status + if last_valid_status is None: + return 0 + # we just calculate how far in the task the agent went during the benchmark time + return (last_valid_status.get_progress() - last_valid_status.get_rejected()) / float(last_valid_status.get_progress_total()) + return 0 + + def stream_watcher(self, identifier, stream): + for line in stream: + self.io_q.put((identifier, line)) + if not stream.closed: + stream.close() + + def run_speed_benchmark(self, task): + args = " --machine-readable --quiet --progress-only" + args += " --restore-disable --potfile-disable --session=hashtopolis -p \"" + str(chr(9)) + "\" " + if 'usePrince' in task and task['usePrince']: + args += get_rules_and_hl(update_files(task['attackcmd']), task['hashlistAlias']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + ' ' + args += " example.dict" + ' ' + task['cmdpars'] + else: + args += update_files(task['attackcmd']).replace(task['hashlistAlias'], "../../hashlists/" + str(task['hashlistId'])) + ' ' + task['cmdpars'] + if 'usePreprocessor' in task and task['usePreprocessor']: + args += " example.dict" + if 'useBrain' in task and task['useBrain']: + args += " -S" + args += " -o ../../hashlists/" + str(task['hashlistId']) + ".out" + full_cmd = self.callPath + args + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + try: + logging.debug("CALL: " + full_cmd) + output = subprocess.check_output(full_cmd, shell=True, cwd=self.cracker_path) + except subprocess.CalledProcessError as e: + logging.error("Error during speed benchmark, return code: " + str(e.returncode)) + send_error("Speed benchmark failed!", self.config.get_value('token'), task['taskId'], None) + return 0 + output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") + benchmark_sum = [0, 0] + for line in output: + if not line: + continue + line = line.split(":") + if len(line) != 3: + continue + # we need to do a weighted sum of all the time outputs of the GPUs + benchmark_sum[0] += int(line[1]) + benchmark_sum[1] += float(line[2])*int(line[1]) + if benchmark_sum[0] == 0: + return 0 # in this case some error happened on the benchmark + return str(benchmark_sum[0]) + ":" + str(float(benchmark_sum[1]) / benchmark_sum[0]) + + def output_watcher(self, file_path, process): + while not os.path.exists(file_path): + if process.poll() is not None: + return + time.sleep(1) + file_handle = open(file_path, encoding="utf-8") + end_count = 0 + while 1: + where = file_handle.tell() + line = file_handle.readline() + if not line: + if process.poll() is None: + time.sleep(0.05) + file_handle.seek(where) + else: + time.sleep(0.05) + end_count += 1 + if end_count > 20: + break + else: + self.lock.acquire() + self.cracks.append(line.strip()) + self.lock.release() + file_handle.close() + + def agent_stopped(self): + return self.wasStopped + + def run_health_check(self, attack, hashlist_alias): + args = " --machine-readable --quiet" + args += " --restore-disable --potfile-disable --session=health " + args += update_files(attack).replace(hashlist_alias, "../../hashlists/health_check.txt") + args += " -o ../../hashlists/health_check.out" + full_cmd = self.callPath + args + if Initialize.get_os() == 1: + full_cmd = full_cmd.replace("/", '\\') + logging.debug("CALL: " + full_cmd) + proc = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) + output, error = proc.communicate() + logging.debug("Started health check attack") + # wait until done, on the health check we don't send any update during running. Maybe later we could at least + # introduce some heartbeat update to make visible that the agent is still alive. + proc.wait() + errors = [] + states = [] + if error: + error = escape_ansi(error.replace(b"\r\n", b"\n").decode('utf-8')) + error = error.split('\n') + for line in error: + if not line: + continue + errors.append(line) + if output: + output = escape_ansi(output.replace(b"\r\n", b"\n").decode('utf-8')) + output = output.split('\n') + for line in output: + if not line: + continue + logging.debug(line) + status = HashcatStatus(line) + if status.is_valid(): + states.append(status) + return [states, errors] diff --git a/htpclient/helpers.py b/htpclient/helpers.py index c0926a0..83e4bd7 100644 --- a/htpclient/helpers.py +++ b/htpclient/helpers.py @@ -109,6 +109,7 @@ def clean_list(element_list): return element_list +# the prince flag is deprecated def update_files(command, prince=False): split = command.split(" ") ret = [] diff --git a/htpclient/initialize.py b/htpclient/initialize.py index 09ca59e..43d355b 100644 --- a/htpclient/initialize.py +++ b/htpclient/initialize.py @@ -16,7 +16,7 @@ def get_version(): @staticmethod def get_version_number(): - return "0.5.0" + return "0.6.0" def run(self, args): self.__check_url(args) @@ -201,3 +201,5 @@ def __build_directories(): os.mkdir("files") if not os.path.isdir("hashlists"): os.mkdir("hashlists") + if not os.path.isdir("preprocessor"): + os.mkdir("preprocessor") diff --git a/htpclient/jsonRequest.py b/htpclient/jsonRequest.py index 8b3bc57..1d68e45 100644 --- a/htpclient/jsonRequest.py +++ b/htpclient/jsonRequest.py @@ -13,7 +13,7 @@ def __init__(self, data): def execute(self): try: logging.debug(self.data) - r = self.session.post(self.config.get_value('url'), json=self.data) + r = self.session.post(self.config.get_value('url'), json=self.data, timeout=30) if r.status_code != 200: logging.error("Status code from server: " + str(r.status_code)) return None diff --git a/htpclient/task.py b/htpclient/task.py index 7ad22af..55283d4 100644 --- a/htpclient/task.py +++ b/htpclient/task.py @@ -11,6 +11,7 @@ def __init__(self): self.taskId = 0 self.task = None self.config = Config() + self.preprocessor = None def reset_task(self): self.task = None @@ -46,3 +47,9 @@ def get_task(self): def get_task_id(self): return self.taskId + + def set_preprocessor(self, settings): + self.preprocessor = settings + + def get_preprocessor(self): + return self.preprocessor