Merge pull request #36 from hashtopolis/bug/windows-path

Adding test framework, fix for window paths and fix for preprocessors

.devcontainer/Dockerfile

@@ -43,7 +43,9 @@ ENV DEBIAN_FRONTEND=dialog
 WORKDIR /app
 USER vscode
 COPY requirements.txt /app/src/
+COPY requirements-tests.txt /app/src/
 RUN /usr/bin/pip3 install -r src/requirements.txt
+RUN /usr/bin/pip3 install -r src/requirements-tests.txt
 
 # Check for and run optional user-supplied command to enable (advanced) customizations of the dev container
 RUN if [ -n "${DEV_CONTAINER_USER_CMD_POST}" ]; then echo "${DEV_CONTAINER_USER_CMD_POST}" | sh ; fi

.devcontainer/windows/Dockerfile

@@ -0,0 +1,20 @@
+FROM mcr.microsoft.com/windows-cssc/python3.7.2server:ltsc2022
+# Nano image doesn't work because some API are not available
+
+# TODO: Support for USER_CMD_PRE and POST?
+# TODO: Create a vscode user?
+# TODO: OpenCL/Nvidia?
+
+WORKDIR C:/App/
+
+# Installing python requirements
+COPY requirements.txt C:/App/
+COPY requirements-tests.txt C:/App/
+RUN pip3 install -r requirements.txt -r requirements-tests.txt
+
+# Fix for host.docker.internal not working
+COPY .devcontainer/windows/entrypoint.ps1 C:/
+COPY .devcontainer/windows/fix-hosts.ps1 C:/
+
+# Setting entrypoint
+ENTRYPOINT "C:\entrypoint.ps1"

.devcontainer/windows/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3"
+services:
+  hashtopolis-agent-windows:
+    container_name: hashtopolis_agent_windows
+    build:
+      context: ../..
+      dockerfile: .devcontainer/windows/Dockerfile
+    volumes:
+      - ../..:C:\App\

.devcontainer/windows/entrypoint.ps1

@@ -0,0 +1,2 @@
+powershell C:\fix-hosts.ps1
+cmd /c ping -t localhost > $null

.devcontainer/windows/fix-hosts.ps1

@@ -0,0 +1,43 @@
+# Source https://github.com/docker/for-win/issues/1976
+# Credits https://github.com/brunnotelma
+$hostsFile = "C:\Windows\System32\drivers\etc\hosts"
+
+try {
+    $DnsEntries = @("host.docker.internal", "gateway.docker.internal")
+    # Tries resolving names for Docker
+    foreach ($Entry in $DnsEntries) {
+        # If any of the names are not resolved, throws an exception
+        Resolve-DnsName -Name $Entry -ErrorAction Stop
+    }
+
+    # If it passes, means that DNS is already configured
+    Write-Host("DNS settings are already configured.")
+} catch {
+    # Gets the gateway IP address, that is the Host's IP address in the Docker network
+    $ip = (ipconfig | where-object {$_ -match "Default Gateway"} | foreach-object{$_.Split(":")[1]}).Trim()
+    # Read the current content from Hosts file
+    $src = [System.IO.File]::ReadAllLines($hostsFile)
+    # Add the a new line after the content
+    $lines = $src += ""
+
+    # Check the hosts file and write it in if its not there...
+    if((cat $hostsFile | Select-String -Pattern "host.docker.internal") -And (cat $hostsFile | Select-String -Pattern "gateway.docker.internal")) {
+        For ($i=0; $i -le $lines.length; $i++) {
+            if ($lines[$i].Contains("host.docker.internal"))
+            {
+                $lines[$i] = ("{0} host.docker.internal" -f $ip)
+                $lines[$i+1] = ("{0} gateway.docker.internal" -f $ip)
+                break
+            }
+        }
+    } else {
+        $lines = $lines += "# Added by Docker for Windows"
+        $lines = $lines += ("{0} host.docker.internal" -f $ip)
+        $lines = $lines += ("{0} gateway.docker.internal" -f $ip)
+        $lines = $lines += "# End of section"
+    }
+    # Writes the new content to the Hosts file
+    [System.IO.File]::WriteAllLines($hostsFile, [string[]]$lines)
+
+    Write-Host("New DNS settings written successfully.")
+}

.gitignore

@@ -1,8 +1,11 @@
 *.iml
 *.exe
+7zr
 *.log
 *.json
+!/tests/*.json
 crackers
+preprocessors
 prince
 files
 hashlists

.vscode/launch.json

@@ -1,4 +1,5 @@
 {
+    "version": "0.2.0",
     "configurations": [
         {
             "name": "Python: Current File",

changelog.md

@@ -1,3 +1,11 @@
+## v0.7.0 -> v0.7.1
+
+### Bugfixes
+
+* Agent working again on Windows, all paths have been converted to Pathlib.Path and using the correct quoting of arguments.
+* 7z wordlist not extracting correctly on Windows.
+* preprocessor not working correctly on both Windows and Linux.
+
 ## v0.6.1 -> v0.7.0
 
 ### Enhancements

htpclient/binarydownload.py

@@ -1,5 +1,6 @@
 import logging
 import os.path
+from pathlib import Path
 import stat
 import sys
 from time import sleep
@@ -135,7 +136,7 @@ def check_prince(self):
 
     def check_preprocessor(self, task):
         logging.debug("Checking if requested preprocessor is present...")
-        path = self.config.get_value('preprocessors-path') + "/" + str(task.get_task()['preprocessor']) + "/"
+        path = Path(self.config.get_value('preprocessors-path'), str(task.get_task()['preprocessor']))
         query = copy_and_set_token(dict_downloadBinary, self.config.get_value('token'))
         query['type'] = 'preprocessor'
         query['preprocessorId'] = task.get_task()['preprocessor']
@@ -160,20 +161,20 @@ def check_preprocessor(self, task):
                 sleep(5)
                 return False
             if Initialize.get_os() == 1:
-                os.system("7zr" + Initialize.get_os_extension() + " x -otemp temp.7z")
+                os.system(f"7zr{Initialize.get_os_extension()} x -otemp temp.7z")
             else:
-                os.system("./7zr" + Initialize.get_os_extension() + " x -otemp temp.7z")
+                os.system(f"./7zr{Initialize.get_os_extension()} x -otemp temp.7z")
             for name in os.listdir("temp"):  # this part needs to be done because it is compressed with the main subfolder of prince
-                if os.path.isdir("temp/" + name):
-                    os.rename("temp/" + name, path)
+                if os.path.isdir(Path('temp', name)):
+                    os.rename(Path('temp', name), path)
                     break
             os.unlink("temp.7z")
             os.rmdir("temp")
             logging.debug("Preprocessor downloaded and extracted")
         return True
 
     def check_version(self, cracker_id):
-        path = self.config.get_value('crackers-path') + "/" + str(cracker_id) + "/"
+        path = Path(self.config.get_value('crackers-path'), str(cracker_id))
         query = copy_and_set_token(dict_downloadBinary, self.config.get_value('token'))
         query['type'] = 'cracker'
         query['binaryVersionId'] = cracker_id
@@ -195,15 +196,28 @@ def check_version(self, cracker_id):
                     logging.error("Download of cracker binary failed!")
                     sleep(5)
                     return False
+
+                # we need to extract the 7zip
+                temp_folder = Path(self.config.get_value('crackers-path'), 'temp')
+                zip_file = Path(self.config.get_value('crackers-path'), f'{cracker_id}.7z')
+
                 if Initialize.get_os() == 1:
-                    os.system("7zr" + Initialize.get_os_extension() + " x -o'" + self.config.get_value('crackers-path') + "/temp' '" + self.config.get_value('crackers-path') + "/" + str(cracker_id) + ".7z'")
+                    # Windows
+                    cmd = f'7zr{Initialize.get_os_extension()} x -o"{temp_folder}" "{zip_file}"'
                 else:
-                    os.system("./7zr" + Initialize.get_os_extension() + " x -o'" + self.config.get_value('crackers-path') + "/temp' '" + self.config.get_value('crackers-path') + "/" + str(cracker_id) + ".7z'")
-                os.unlink(self.config.get_value('crackers-path') + "/" + str(cracker_id) + ".7z")
-                for name in os.listdir(self.config.get_value('crackers-path') + "/temp"):
-                    if os.path.isdir(self.config.get_value('crackers-path') + "/temp/" + name):
-                        os.rename(self.config.get_value('crackers-path') + "/temp/" + name, self.config.get_value('crackers-path') + "/" + str(cracker_id))
+                    # Linux
+                    cmd = f"./7zr{Initialize.get_os_extension()} x -o'{temp_folder}' '{zip_file}'"
+                os.system(cmd)
+
+                # Clean up 7zip
+                os.unlink(zip_file)
+
+                # Workaround for a 7zip containing a folder name or already the contents of a cracker
+                for name in os.listdir(temp_folder):
+                    to_check_path = Path(temp_folder, name)
+                    if os.path.isdir(to_check_path):
+                        os.rename(to_check_path, path)
                     else:
-                        os.rename(self.config.get_value('crackers-path') + "/temp", self.config.get_value('crackers-path') + "/" + str(cracker_id))
+                        os.rename(temp_folder, path)
                         break
         return True

htpclient/download.py

@@ -14,8 +14,6 @@ class Download:
     def download(url, output, no_header=False):
         try:
             session = Session().s
-            if Initialize.get_os() == 1:
-                output = output.replace("/", '\\')
 
             # Check header
             if not no_header:

htpclient/files.py

@@ -1,6 +1,7 @@
 import logging
 import time
 from time import sleep
+from pathlib import Path
 
 import os
 
@@ -36,26 +37,33 @@ def deletion_check(self):
         else:
             files = ans['filenames']
             for filename in files:
+                file_path = Path(self.config.get_value('files-path'), filename)
                 if filename.find("/") != -1 or filename.find("\\") != -1:
                     continue  # ignore invalid file names
-                elif os.path.dirname(self.config.get_value('files-path') + "/" + filename) != "files":
+                elif os.path.dirname(file_path) != "files":
                     continue  # ignore any case in which we would leave the files folder
-                elif os.path.exists(self.config.get_value('files-path') + "/" + filename):
+                elif os.path.exists(file_path):
                     logging.info("Delete file '" + filename + "' as requested by server...")
-                    if os.path.splitext(self.config.get_value('files-path') + "/" + filename)[1] == '.7z':
-                        if os.path.exists(self.config.get_value('files-path') + "/" + filename.replace(".7z", ".txt")):
+                    # When we get the delete requests, this function will check if the <filename>.7z maybe as
+                    # an extracted text file. That file will also be deleted.
+                    if os.path.splitext(file_path)[1] == '.7z':
+                        txt_file = Path(f"{os.path.splitext(file_path)[0]}.txt")
+                        if os.path.exists(txt_file):
                             logging.info("Also delete assumed wordlist from archive of same file...")
-                            os.unlink(self.config.get_value('files-path') + "/" + filename.replace(".7z", ".txt"))
-                    os.unlink(self.config.get_value('files-path') + "/" + filename)
+                            os.unlink(txt_file)
+                    os.unlink(file_path)
 
     def check_files(self, files, task_id):
         for file in files:
-            file_localpath = self.config.get_value('files-path') + "/" + file
+            file_localpath = Path(self.config.get_value('files-path'), file)
+            txt_file = Path(f"{os.path.splitext(file_localpath)[0]}.txt")
             query = copy_and_set_token(dict_getFile, self.config.get_value('token'))
             query['taskId'] = task_id
             query['file'] = file
             req = JsonRequest(query)
             ans = req.execute()
+
+            # Process request
             if ans is None:
                 logging.error("Failed to get file!")
                 sleep(5)
@@ -65,30 +73,44 @@ def check_files(self, files, task_id):
                 sleep(5)
                 return False
             else:
+                # Filesize is OK
                 file_size = int(ans['filesize'])
                 if os.path.isfile(file_localpath) and os.stat(file_localpath).st_size == file_size:
                     logging.debug("File is present on agent and has matching file size.")
                     continue
+
+                # Multicasting configured
                 elif self.config.get_value('multicast'):
                     logging.debug("Multicast is enabled, need to wait until it was delivered!")
                     sleep(5)  # in case the file is not there yet (or not completely), we just wait some time and then try again
                     return False
+
                 # TODO: we might need a better check for this
-                if os.path.isfile(file_localpath.replace(".7z", ".txt")):
+                if os.path.isfile(txt_file):
                     continue
+
+                # Rsync
                 if self.config.get_value('rsync') and Initialize.get_os() != 1:
-                    Download.rsync(self.config.get_value('rsync-path') + '/' + file, file_localpath)
+                    Download.rsync(Path(self.config.get_value('rsync-path'), file), file_localpath) 
                 else:
                     logging.debug("Starting download of file from server...")
                     Download.download(self.config.get_value('url').replace("api/server.php", "") + ans['url'], file_localpath)
+
+                # Mismatch filesize
                 if os.path.isfile(file_localpath) and os.stat(file_localpath).st_size != file_size:
                     logging.error("file size mismatch on file: %s" % file)
                     sleep(5)
                     return False
-                if os.path.splitext(self.config.get_value('files-path') + "/" + file)[1] == '.7z' and not os.path.isfile(self.config.get_value('files-path') + "/" + file.replace(".7z", ".txt")):
+
+                # 7z extraction, check if the <filename>.txt does exist.
+                if os.path.splitext(file_localpath)[1] == '.7z' and not os.path.isfile(txt_file):
                     # extract if needed
-                    if Initialize.get_os() != 1:
-                        os.system("./7zr" + Initialize.get_os_extension() + " x -aoa -o'" + self.config.get_value('files-path') + "/' -y '" + self.config.get_value('files-path') + "/" + file + "'")
+                    files_path = Path(self.config.get_value('files-path'))
+                    if Initialize.get_os() == 1:
+                        # Windows
+                        cmd = f'7zr{Initialize.get_os_extension()} x -aoa -o"{files_path}" -y "{file_localpath}"'
                     else:
-                        os.system("7zr" + Initialize.get_os_extension() + " x -aoa -o'" + self.config.get_value('files-path') + "/' -y '" + self.config.get_value('files-path') + "/" + file + "'")
+                        # Linux
+                        cmd = f"./7zr{Initialize.get_os_extension()} x -aoa -o'{files_path}' -y '{file_localpath}'"
+                    os.system(cmd)
         return True